Compare commits
90 Commits
master
...
releases/2
| Author | SHA1 | Date |
|---|---|---|
|
|
b4d9b68609 | |
|
|
a8984f3761 | |
|
|
a8a6e8f60a | |
|
|
71ebd41475 | |
|
|
3e8f2cc58b | |
|
|
e8c1a59329 | |
|
|
b371b52680 | |
|
|
7ea2d668ad | |
|
|
07f95e37b1 | |
|
|
cd9aece3d7 | |
|
|
2e6a746722 | |
|
|
b37376100a | |
|
|
0ed9857b97 | |
|
|
6f801e5b34 | |
|
|
5d5bbce492 | |
|
|
4795391b73 | |
|
|
f507ac0364 | |
|
|
3347f930cb | |
|
|
4dfeba454b | |
|
|
f128545a2a | |
|
|
2376b3b1a4 | |
|
|
ca6f8fc9cf | |
|
|
c6f2f9739c | |
|
|
7d58dd59b6 | |
|
|
5e81820a5b | |
|
|
1d8112bba6 | |
|
|
08794e70fe | |
|
|
99384c7846 | |
|
|
fc1e019ff5 | |
|
|
8b4a953993 | |
|
|
99c746f2c3 | |
|
|
6b8f778d46 | |
|
|
3d48b7f078 | |
|
|
5c683584fc | |
|
|
cf3e15bed5 | |
|
|
106d5653c2 | |
|
|
39447e08e7 | |
|
|
176bdf5137 | |
|
|
f730bf9ae9 | |
|
|
422127e33d | |
|
|
1e1eafe09d | |
|
|
d2e3e1fc00 | |
|
|
2a57c7bbe4 | |
|
|
10274c0908 | |
|
|
cb71090389 | |
|
|
0335cdbbc6 | |
|
|
ff878ffe92 | |
|
|
8f507a07f5 | |
|
|
a09de25dd2 | |
|
|
86f96a8f3a | |
|
|
6966e383d3 | |
|
|
ab1a3e8d84 | |
|
|
29d9462a9b | |
|
|
c067ed1b9a | |
|
|
bff2fe0b2f | |
|
|
fd532c2611 | |
|
|
1d68caf96f | |
|
|
7bd3738afd | |
|
|
5767a54fa2 | |
|
|
56326a3d2d | |
|
|
1baf72a780 | |
|
|
c22e65e8a6 | |
|
|
3964168205 | |
|
|
a416a80c86 | |
|
|
81152e07f3 | |
|
|
a14fdef9b6 | |
|
|
ce3bea3c07 | |
|
|
707a0833d8 | |
|
|
71cde81827 | |
|
|
f4ed2572a5 | |
|
|
ec02bc4dbd | |
|
|
5a05e52d40 | |
|
|
fee727f415 | |
|
|
01c3951887 | |
|
|
7715d033d2 | |
|
|
1da3539bc1 | |
|
|
ec8527f21e | |
|
|
1c35875a60 | |
|
|
1e392b0180 | |
|
|
a05b7c76b2 | |
|
|
3a9c731c2b | |
|
|
641f1425df | |
|
|
e67dffa6a3 | |
|
|
7b85eef15f | |
|
|
ea0f6fce5a | |
|
|
6959f6671f | |
|
|
322bc3498b | |
|
|
6effe67358 | |
|
|
759c91d547 | |
|
|
5abbe2fec5 |
|
|
@ -0,0 +1,161 @@
|
|||
jobs:
|
||||
- job: Lin
|
||||
# About 150% of total time
|
||||
timeoutInMinutes: 90
|
||||
|
||||
pool:
|
||||
name: LIN_VMSS_VENV_F8S_WU2
|
||||
|
||||
variables:
|
||||
system.debug: true
|
||||
VSTS_HTTP_RETRY: 5
|
||||
VSTS_HTTP_TIMEOUT: 200
|
||||
WORKERS_NUMBER: 8
|
||||
BUILD_TYPE: Release
|
||||
REPO_DIR: $(Build.Repository.LocalPath)
|
||||
WORK_DIR: $(Pipeline.Workspace)/_w
|
||||
BUILD_DIR: $(WORK_DIR)/build
|
||||
BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
|
||||
whoami
|
||||
uname -a
|
||||
which python3
|
||||
python3 --version
|
||||
which java
|
||||
java -version
|
||||
gcc --version
|
||||
lsb_release
|
||||
env
|
||||
cat /proc/cpuinfo
|
||||
cat /proc/meminfo
|
||||
cat /etc/fstab
|
||||
vmstat -s
|
||||
df
|
||||
lsblk -o NAME,HCTL,SIZE,MOUNTPOINT | grep -i "sd"
|
||||
displayName: 'System info'
|
||||
|
||||
- script: |
|
||||
rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
|
||||
rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
|
||||
displayName: 'Make dir'
|
||||
|
||||
- checkout: self
|
||||
clean: true
|
||||
lfs: false
|
||||
submodules: recursive
|
||||
path: openvino
|
||||
|
||||
- script: |
|
||||
sudo apt --assume-yes install libusb-1.0-0-dev
|
||||
python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/requirements.txt
|
||||
# For running Python API tests
|
||||
python3 -m pip install -r $(REPO_DIR)/inference-engine/ie_bridges/python/src/requirements-dev.txt
|
||||
# Speed up build
|
||||
wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
|
||||
unzip ninja-linux.zip
|
||||
sudo cp -v ninja /usr/local/bin/
|
||||
# Speed up tests
|
||||
git clone https://github.com/google/gtest-parallel.git
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Install dependencies'
|
||||
|
||||
- task: CMake@1
|
||||
inputs:
|
||||
# CMake must get Python 3.x version by default
|
||||
cmakeArgs: -GNinja -DVERBOSE_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DENABLE_TESTS=ON $(REPO_DIR)
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
|
||||
- script: ninja
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Build Lin'
|
||||
|
||||
- script: ls -alR $(REPO_DIR)/bin/
|
||||
displayName: 'List files'
|
||||
|
||||
- script: $(BIN_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
|
||||
displayName: 'nGraph UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/InferenceEngineUnitTests --gtest_print_time=1 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml
|
||||
displayName: 'IE UT old'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/ieUnitTests --gtest_output=xml:TEST-ieUnitTests.xml
|
||||
displayName: 'IE UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/cpuUnitTests --gtest_output=xml:TEST-cpuUnitTests.xml
|
||||
displayName: 'CPU UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/gnaUnitTests --gtest_output=xml:TEST-gnaUnitTests.xml
|
||||
displayName: 'GNA UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/vpuUnitTests --gtest_output=xml:TEST-vpuUnitTests.xml
|
||||
displayName: 'VPU UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/onnxImporterUnitTests --gtest_output=xml:TEST-onnxImporterUnitTests.xml
|
||||
displayName: 'ONNX Importer UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/ieFuncTests --gtest_output=xml:TEST-ieFuncTests.xml
|
||||
displayName: 'IE FuncTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml
|
||||
displayName: 'CPU FuncTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/MklDnnBehaviorTests --gtest_output=xml:TEST-MklDnnBehaviorTests.xml
|
||||
displayName: 'MklDnnBehaviorTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
git clone https://github.com/openvinotoolkit/testdata.git
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Clone testdata'
|
||||
|
||||
- script: |
|
||||
export DATA_PATH=$(WORK_DIR)/testdata
|
||||
export MODELS_PATH=$(WORK_DIR)/testdata
|
||||
python3 $(WORK_DIR)/gtest-parallel/gtest-parallel $(BIN_DIR)/MklDnnFunctionalTests --workers=$(WORKERS_NUMBER) --dump_json_test_results=MklDnnFunctionalTests.json --gtest_filter=*smoke* -- --gtest_print_time=1
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'MklDnnFunctionalTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
export DATA_PATH=$(WORK_DIR)/testdata
|
||||
export MODELS_PATH=$(WORK_DIR)/testdata
|
||||
$(BIN_DIR)/InferenceEngineCAPITests --gtest_output=xml:TEST-InferenceEngineCAPITests.xml
|
||||
displayName: 'IE CAPITests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
export DATA_PATH=$(WORK_DIR)/testdata
|
||||
export MODELS_PATH=$(WORK_DIR)/testdata
|
||||
export LD_LIBRARY_PATH=$(BIN_DIR)/lib
|
||||
export PYTHONPATH=$(BIN_DIR)/lib/python_api/python3.6
|
||||
env
|
||||
cd $(REPO_DIR)/inference-engine/ie_bridges/python/tests
|
||||
pytest pytest --junitxml=TEST-PythonAPI.xml
|
||||
displayName: 'Python API Tests'
|
||||
continueOnError: false
|
||||
enabled: false
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: always()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit, cTest
|
||||
testResultsFiles: '**/TEST-*.xml'
|
||||
#searchFolder: '$(BUILD_DIR)'
|
||||
mergeTestResults: false # Optional
|
||||
#failTaskOnFailedTests: false # Optional
|
||||
#testRunTitle: 'Pre/Post-Commit' # Optional
|
||||
buildPlatform: 'x64' # Optional
|
||||
buildConfiguration: 'Linux' # Optional
|
||||
#publishRunAttachments: true # Optional
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
jobs:
|
||||
- job: nGraph_ONNX_Lin
|
||||
|
||||
# About 150% of total time
|
||||
timeoutInMinutes: 60
|
||||
|
||||
pool:
|
||||
name: LIN_VMSS_VENV_F8S_WU2
|
||||
|
||||
variables:
|
||||
system.debug: true
|
||||
VSTS_HTTP_RETRY: 5
|
||||
VSTS_HTTP_TIMEOUT: 200
|
||||
WORKERS_NUMBER: 8
|
||||
BUILD_TYPE: Release
|
||||
REPO_DIR: $(Build.Repository.LocalPath)
|
||||
WORK_DIR: $(Pipeline.Workspace)/_w
|
||||
BUILD_DIR: $(WORK_DIR)/build
|
||||
BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
|
||||
INSTALL_DIR: $(WORK_DIR)/install
|
||||
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
lfs: false
|
||||
submodules: recursive
|
||||
path: openvino
|
||||
|
||||
- script: |
|
||||
curl -H Metadata:true --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2019-06-01"
|
||||
whoami
|
||||
uname -a
|
||||
which python3
|
||||
python3 --version
|
||||
gcc --version
|
||||
lsb_release
|
||||
env
|
||||
cat /proc/cpuinfo
|
||||
cat /proc/meminfo
|
||||
vmstat -s
|
||||
df
|
||||
displayName: 'System info'
|
||||
|
||||
- script: |
|
||||
rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
|
||||
displayName: 'Make dir'
|
||||
|
||||
- script: |
|
||||
sudo apt --assume-yes install libusb-1.0-0-dev
|
||||
python3 -m pip install -r ./inference-engine/ie_bridges/python/requirements.txt
|
||||
# For running Python API tests
|
||||
python3 -m pip install -r ./inference-engine/ie_bridges/python/src/requirements-dev.txt
|
||||
displayName: 'Install dependencies'
|
||||
enabled: false
|
||||
|
||||
- script: |
|
||||
wget https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip
|
||||
unzip ninja-linux.zip
|
||||
sudo cp -v ninja /usr/local/bin/
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Install Ninja'
|
||||
enabled: false
|
||||
|
||||
- task: CMake@1
|
||||
inputs:
|
||||
# CMake must get Python 3.x version by default
|
||||
cmakeArgs: -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_VPU=OFF -DENABLE_GNA=OFF -DENABLE_OPENCV=OFF -DENABLE_CPPLINT=OFF -DENABLE_TESTS=OFF -DENABLE_BEH_TESTS=OFF -DENABLE_FUNCTIONAL_TESTS=OFF -DENABLE_MKL_DNN=ON -DENABLE_CLDNN=OFF -DENABLE_PROFILING_ITT=OFF -DENABLE_SAMPLES=OFF -DENABLE_SPEECH_DEMO=OFF -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3.6 -DNGRAPH_ONNX_IMPORT_ENABLE=ON -DNGRAPH_INTERPRETER_ENABLE=ON -DNGRAPH_DEBUG_ENABLE=OFF -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) $(REPO_DIR)
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
enabled: false
|
||||
|
||||
- script: ninja
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Build'
|
||||
enabled: false
|
||||
|
||||
- script: make install
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Install'
|
||||
enabled: false
|
||||
|
||||
- script: |
|
||||
ls -alR $(REPO_DIR)/bin/
|
||||
ls -alR $(INSTALL_DIR)
|
||||
displayName: 'List files'
|
||||
enabled: false
|
||||
|
||||
- script: docker build --tag=openvino-onnx-ci-image --file=$(REPO_DIR)/.ci/openvino-onnx/Dockerfile .
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Docker build'
|
||||
enabled: false
|
||||
|
||||
- script: docker run --name openvino-onnx-ci-container openvino-onnx-ci-image
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Docker run tests'
|
||||
enabled: false
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
jobs:
|
||||
- job: Mac
|
||||
# About 200% of total time (perfomace of Mac hosts is unstable)
|
||||
timeoutInMinutes: 240
|
||||
|
||||
pool:
|
||||
vmImage: 'macOS-10.15'
|
||||
|
||||
variables:
|
||||
system.debug: true
|
||||
VSTS_HTTP_RETRY: 5
|
||||
VSTS_HTTP_TIMEOUT: 200
|
||||
WORKERS_NUMBER: 3
|
||||
BUILD_TYPE: Release
|
||||
REPO_DIR: $(Build.Repository.LocalPath)
|
||||
WORK_DIR: $(Pipeline.Workspace)/_w
|
||||
BUILD_DIR: $(WORK_DIR)/build
|
||||
BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
whoami
|
||||
uname -a
|
||||
which python3
|
||||
python3 --version
|
||||
which java
|
||||
java -version
|
||||
gcc --version
|
||||
xcrun --sdk macosx --show-sdk-version
|
||||
env
|
||||
sysctl -a
|
||||
displayName: 'System info'
|
||||
|
||||
- script: |
|
||||
rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
|
||||
rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
|
||||
displayName: 'Make dir'
|
||||
|
||||
- checkout: self
|
||||
clean: true
|
||||
lfs: false
|
||||
submodules: recursive
|
||||
path: openvino
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.7'
|
||||
|
||||
- script: |
|
||||
brew install cython
|
||||
brew install automake
|
||||
# Speed up build
|
||||
brew install ninja
|
||||
# Speed up tests
|
||||
git clone https://github.com/google/gtest-parallel.git
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Install dependencies'
|
||||
|
||||
- script: |
|
||||
export PATH="/usr/local/opt/cython/bin:$PATH"
|
||||
export CC=gcc
|
||||
export CXX=g++
|
||||
# Disable errors with Ninja
|
||||
export CXXFLAGS="-Wno-error=unused-command-line-argument"
|
||||
export CFLAGS="-Wno-error=unused-command-line-argument"
|
||||
cmake -GNinja -DVERBOSE_BUILD=ON -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_PYTHON=ON -DENABLE_TESTS=ON $(REPO_DIR)
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'CMake'
|
||||
|
||||
- script: ninja
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Build Mac'
|
||||
|
||||
- script: ls -alR $(REPO_DIR)/bin/
|
||||
displayName: 'List files'
|
||||
|
||||
- script: $(BIN_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU*:IE_CPU.onnx_model_sigmoid --gtest_output=xml:TEST-NGraphUT.xml
|
||||
displayName: 'nGraph UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/InferenceEngineUnitTests --gtest_print_time=1 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml
|
||||
displayName: 'IE UT old'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/ieUnitTests --gtest_output=xml:TEST-ieUnitTests.xml
|
||||
displayName: 'IE UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/cpuUnitTests --gtest_output=xml:TEST-cpuUnitTests.xml
|
||||
displayName: 'CPU UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/vpuUnitTests --gtest_output=xml:TEST-vpuUnitTests.xml
|
||||
displayName: 'VPU UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/onnxImporterUnitTests --gtest_output=xml:TEST-onnxImporterUnitTests.xml
|
||||
displayName: 'ONNX Importer UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/ieFuncTests --gtest_output=xml:TEST-ieFuncTests.xml
|
||||
displayName: 'IE FuncTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml
|
||||
displayName: 'CPU FuncTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: $(BIN_DIR)/MklDnnBehaviorTests --gtest_output=xml:TEST-MklDnnBehaviorTests.xml
|
||||
displayName: 'MklDnnBehaviorTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
git clone --single-branch --branch releases/2021/2 https://github.com/openvinotoolkit/testdata.git
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Clone testdata'
|
||||
|
||||
- script: |
|
||||
export DATA_PATH=$(WORK_DIR)/testdata
|
||||
export MODELS_PATH=$(WORK_DIR)/testdata
|
||||
python3 $(WORK_DIR)/gtest-parallel/gtest-parallel $(BIN_DIR)/MklDnnFunctionalTests --workers=$(WORKERS_NUMBER) --dump_json_test_results=MklDnnFunctionalTests.json --gtest_filter=*smoke*:-smoke_MobileNet/ModelTransformationsTest.LPT/mobilenet_v2_tf_depthwise_batch1_inPluginDisabled_inTestDisabled_asymmetric* -- --gtest_print_time=1
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'MklDnnFunctionalTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
export DATA_PATH=$(WORK_DIR)/testdata
|
||||
export MODELS_PATH=$(WORK_DIR)/testdata
|
||||
$(BIN_DIR)/InferenceEngineCAPITests --gtest_output=xml:TEST-InferenceEngineCAPITests.xml
|
||||
displayName: 'IE CAPITests'
|
||||
continueOnError: false
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: always()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit, cTest
|
||||
testResultsFiles: '**/TEST-*.xml'
|
||||
#searchFolder: '$(BUILD_DIR)'
|
||||
mergeTestResults: false # Optional
|
||||
#failTaskOnFailedTests: false # Optional
|
||||
#testRunTitle: 'Pre/Post-Commit' # Optional
|
||||
buildPlatform: 'x64' # Optional
|
||||
buildConfiguration: 'Mac' # Optional
|
||||
#publishRunAttachments: true # Optional
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
jobs:
|
||||
- job: Win
|
||||
# About 150% of total time
|
||||
timeoutInMinutes: 120
|
||||
|
||||
pool:
|
||||
name: WIN_VMSS_VENV_F8S_WU2
|
||||
|
||||
variables:
|
||||
system.debug: true
|
||||
VSTS_HTTP_RETRY: 5
|
||||
VSTS_HTTP_TIMEOUT: 200
|
||||
WORKERS_NUMBER: 8
|
||||
BUILD_TYPE: Release
|
||||
REPO_DIR: $(Build.Repository.LocalPath)
|
||||
WORK_DIR: $(Pipeline.Workspace)\_w
|
||||
BUILD_DIR: D:\build
|
||||
BIN_DIR: $(REPO_DIR)\bin\intel64
|
||||
MSVS_VARS_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
|
||||
MSVC_COMPILER_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC\14.24.28314\bin\Hostx64\x64\cl.exe
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
powershell -command "Invoke-RestMethod -Headers @{\"Metadata\"=\"true\"} -Method GET -Uri http://169.254.169.254/metadata/instance/compute?api-version=2019-06-01 | format-custom"
|
||||
where python3
|
||||
where python
|
||||
python --version
|
||||
where java
|
||||
java -version
|
||||
wmic computersystem get TotalPhysicalMemory
|
||||
wmic cpu list
|
||||
wmic logicaldisk get description,name
|
||||
wmic VOLUME list
|
||||
set
|
||||
displayName: 'System info'
|
||||
|
||||
- script: |
|
||||
rd /Q /S $(WORK_DIR) & mkdir $(WORK_DIR)
|
||||
rd /Q /S $(BUILD_DIR) & mkdir $(BUILD_DIR)
|
||||
displayName: 'Make dir'
|
||||
|
||||
- checkout: self
|
||||
clean: true
|
||||
lfs: false
|
||||
submodules: recursive
|
||||
path: openvino
|
||||
|
||||
- script: |
|
||||
certutil -urlcache -split -f https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip ninja-win.zip
|
||||
powershell -command "Expand-Archive -Force ninja-win.zip"
|
||||
git clone https://github.com/google/gtest-parallel.git
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Install dependencies'
|
||||
|
||||
- script: |
|
||||
certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/IBSetupConsole_9_5_0.exe IBSetupConsole_9_5_0.exe
|
||||
call IBSetupConsole_9_5_0.exe /Install /Components=Agent,oneuse /Coordinator=11.1.0.4 /AGENT:OPENFIREWALL=ON /AGENT:AUTOSELECTPORTS=ON /ADDTOPATH=ON /AGENT:INSTALLADDINS=OFF
|
||||
workingDirectory: $(WORK_DIR)
|
||||
displayName: 'Install IncrediBuild'
|
||||
|
||||
- script: |
|
||||
echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent
|
||||
reg add HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Xoreax\IncrediBuild\Builder /f /v LastEnabled /d 0 && echo Start IncrediBuild_Agent && net start IncrediBuild_Agent
|
||||
displayName: 'Start IncrediBuild'
|
||||
|
||||
- script: |
|
||||
set PATH=$(WORK_DIR)\ninja-win;%PATH%
|
||||
call "$(MSVS_VARS_PATH)" && cmake -GNinja -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) -DENABLE_TESTS=ON -DCMAKE_C_COMPILER:PATH="$(MSVC_COMPILER_PATH)" -DCMAKE_CXX_COMPILER:PATH="$(MSVC_COMPILER_PATH)" $(REPO_DIR)
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'CMake'
|
||||
|
||||
- script: |
|
||||
set PATH=$(WORK_DIR)\ninja-win;%PATH%
|
||||
call "$(MSVS_VARS_PATH)" && "C:\Program Files (x86)\IncrediBuild\BuildConsole.exe" /COMMAND="ninja" /MaxCPUS=40
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Build Win'
|
||||
|
||||
- script: echo Stop IncrediBuild_Agent && net stop IncrediBuild_Agent
|
||||
displayName: Stop IncrediBuild
|
||||
continueOnError: true
|
||||
- script: dir $(REPO_DIR)\bin\ /s
|
||||
displayName: 'List files'
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
|
||||
displayName: 'nGraph UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\InferenceEngineUnitTests --gtest_print_time=1 --gtest_output=xml:TEST-InferenceEngineUnitTests.xml
|
||||
displayName: 'IE UT old'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\ieUnitTests --gtest_output=xml:TEST-ieUnitTests.xml
|
||||
displayName: 'IE UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\cpuUnitTests --gtest_output=xml:TEST-cpuUnitTests.xml
|
||||
displayName: 'CPU UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\gnaUnitTests --gtest_output=xml:TEST-gnaUnitTests.xml
|
||||
displayName: 'GNA UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\vpuUnitTests --gtest_output=xml:TEST-vpuUnitTests.xml
|
||||
displayName: 'VPU UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\onnxImporterUnitTests --gtest_output=xml:TEST-onnxImporterUnitTests.xml
|
||||
displayName: 'ONNX Importer UT'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\ieFuncTests --gtest_output=xml:TEST-ieFuncTests.xml
|
||||
displayName: 'IE FuncTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\cpuFuncTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-cpuFuncTests.xml
|
||||
displayName: 'CPU FuncTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;%PATH%
|
||||
$(BIN_DIR)\MklDnnBehaviorTests --gtest_output=xml:TEST-MklDnnBehaviorTests.xml
|
||||
displayName: 'MklDnnBehaviorTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
git clone https://github.com/openvinotoolkit/testdata.git
|
||||
workingDirectory: $(BUILD_DIR)
|
||||
displayName: 'Clone testdata'
|
||||
|
||||
# Add for gtest-parallel, it hangs now (CVS-33386)
|
||||
#python $(BUILD_DIR)\gtest-parallel\gtest-parallel $(BIN_DIR)\MklDnnFunctionalTests --workers=$(WORKERS_NUMBER) --dump_json_test_results=MklDnnFunctionalTests.json --gtest_filter=*smoke* -- --gtest_print_time=1
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;$(REPO_DIR)\inference-engine\temp\opencv_4.5.0\opencv\bin;%PATH%
|
||||
set DATA_PATH=$(BUILD_DIR)\testdata
|
||||
set MODELS_PATH=$(BUILD_DIR)\testdata
|
||||
$(BIN_DIR)\MklDnnFunctionalTests --gtest_filter=*smoke* --gtest_print_time=1 --gtest_output=xml:TEST-MklDnnFunctionalTests.xml
|
||||
displayName: 'MklDnnFunctionalTests'
|
||||
continueOnError: false
|
||||
|
||||
- script: |
|
||||
set PATH=$(REPO_DIR)\inference-engine\temp\tbb\bin;$(REPO_DIR)\inference-engine\temp\opencv_4.5.0\opencv\bin;%PATH%
|
||||
set DATA_PATH=$(BUILD_DIR)\testdata
|
||||
set MODELS_PATH=$(BUILD_DIR)\testdata
|
||||
$(BIN_DIR)\InferenceEngineCAPITests --gtest_output=xml:TEST-InferenceEngineCAPITests.xml
|
||||
displayName: 'IE CAPITests'
|
||||
continueOnError: false
|
||||
|
||||
- task: PublishTestResults@2
|
||||
condition: always()
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit, cTest
|
||||
testResultsFiles: '**/TEST-*.xml'
|
||||
#searchFolder: '$(BUILD_DIR)'
|
||||
mergeTestResults: false # Optional
|
||||
#failTaskOnFailedTests: false # Optional
|
||||
#testRunTitle: 'Pre/Post-Commit' # Optional
|
||||
buildPlatform: 'x64' # Optional
|
||||
buildConfiguration: 'Windows' # Optional
|
||||
#publishRunAttachments: true # Optional
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
FROM ubuntu:20.04
|
||||
|
||||
LABEL version=2020.07.09.1
|
||||
|
||||
ARG http_proxy
|
||||
ARG https_proxy
|
||||
ENV http_proxy ${http_proxy}
|
||||
ENV https_proxy ${https_proxy}
|
||||
|
||||
ENV CI=true
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
|
||||
# Install base dependencies
|
||||
RUN apt-get update && apt-get install -y locales && apt-get clean autoclean && apt-get autoremove -y
|
||||
|
||||
# Set the locale to en_US.UTF-8
|
||||
RUN locale-gen en_US.UTF-8
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LANGUAGE en_US:en
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
RUN apt-get update && apt-get -y --no-install-recommends install \
|
||||
# OpenVINO dependencies
|
||||
autoconf \
|
||||
automake \
|
||||
build-essential \
|
||||
cmake \
|
||||
curl \
|
||||
git \
|
||||
libtool \
|
||||
ocl-icd-opencl-dev \
|
||||
pkg-config \
|
||||
unzip \
|
||||
wget \
|
||||
# Python dependencies
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-virtualenv \
|
||||
cython3 \
|
||||
tox \
|
||||
# ONNX dependencies
|
||||
git-lfs \
|
||||
protobuf-compiler \
|
||||
libprotobuf-dev && \
|
||||
apt-get clean autoclean && \
|
||||
apt-get autoremove -y
|
||||
|
||||
# Build OpenVINO
|
||||
COPY . /openvino/
|
||||
WORKDIR /openvino/build
|
||||
RUN cmake .. \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DENABLE_VPU=OFF \
|
||||
-DENABLE_GNA=OFF \
|
||||
-DENABLE_OPENCV=OFF \
|
||||
-DENABLE_CPPLINT=OFF \
|
||||
-DENABLE_TESTS=OFF \
|
||||
-DENABLE_BEH_TESTS=OFF \
|
||||
-DENABLE_FUNCTIONAL_TESTS=OFF \
|
||||
-DENABLE_MKL_DNN=ON \
|
||||
-DENABLE_CLDNN=OFF \
|
||||
-DENABLE_PROFILING_ITT=OFF \
|
||||
-DENABLE_SAMPLES=OFF \
|
||||
-DENABLE_SPEECH_DEMO=OFF \
|
||||
-DENABLE_PYTHON=ON \
|
||||
-DPYTHON_EXECUTABLE=/usr/bin/python3 \
|
||||
-DNGRAPH_ONNX_IMPORT_ENABLE=ON \
|
||||
-DNGRAPH_INTERPRETER_ENABLE=ON \
|
||||
-DNGRAPH_DEBUG_ENABLE=OFF \
|
||||
-DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON \
|
||||
-DCMAKE_INSTALL_PREFIX=/openvino/dist
|
||||
RUN make -j $(nproc) install
|
||||
|
||||
# Run tests via tox
|
||||
WORKDIR /openvino/ngraph/python
|
||||
ENV NGRAPH_CPP_BUILD_PATH=/openvino/dist
|
||||
ENV LD_LIBRARY_PATH=/openvino/dist/lib
|
||||
ENV NGRAPH_ONNX_IMPORT_ENABLE=TRUE
|
||||
ENV PYTHONPATH=/openvino/bin/intel64/Release/lib/python_api/python3.8:${PYTHONPATH}
|
||||
RUN git clone --recursive https://github.com/pybind/pybind11.git -b v2.5.0 --depth 1
|
||||
CMD tox
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
// Copyright (C) 2018-2020 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
DOCKER_CONTAINER_NAME= "openvino-onnx-ci-container"
|
||||
DOCKER_IMAGE_TAG = "openvino-onnx-ci-image"
|
||||
|
||||
// workaround for aborting previous builds on PR update
|
||||
@NonCPS
|
||||
def stopPreviousRunningBuilds() {
|
||||
def jobname = env.JOB_NAME
|
||||
if (jobname.startsWith("onnx/openvino_ci/PR")){
|
||||
def buildnum = env.BUILD_NUMBER.toInteger()
|
||||
def job = Jenkins.instance.getItemByFullName(jobname)
|
||||
def job_newest = job.builds.first()
|
||||
for (build in job.builds.reverse()[0..<-1]) {
|
||||
if (build.isBuilding()){
|
||||
echo "Stop task = ${build} because newest #${job_newest} is on the way"
|
||||
build.doStop();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def getGitPrInfo(String project) {
|
||||
def gitPrInfo = [
|
||||
prAuthorEmail : "",
|
||||
commitAuthorEmail : "",
|
||||
commitHash : "",
|
||||
commitSubject : ""
|
||||
]
|
||||
try {
|
||||
dir ("${WORKDIR}/${project}") {
|
||||
gitPrInfo.prAuthorEmail = sh (script: 'git log -1 --pretty="format:%ae" ', returnStdout: true).trim()
|
||||
gitPrInfo.commitAuthorEmail = sh (script: 'git log -1 --pretty="format:%ce" ', returnStdout: true).trim()
|
||||
gitPrInfo.commitSubject = sh (script: 'git log -1 --pretty="format:%H" ', returnStdout: true).trim()
|
||||
gitPrInfo.commitHash = sh (script: 'git log -1 --pretty="format:%s" ', returnStdout: true).trim()
|
||||
}
|
||||
}
|
||||
catch(e) {
|
||||
echo "Failed to retrieve ${project} git repository information!"
|
||||
echo "ERROR: ${e}"
|
||||
}
|
||||
return gitPrInfo
|
||||
}
|
||||
|
||||
def notifyByEmail(def gitPrInfo) {
|
||||
stage('Notify') {
|
||||
String notifyPeople = "${gitPrInfo.prAuthorEmail}, ${gitPrInfo.commitAuthorEmail}"
|
||||
emailext (
|
||||
subject: "OpenVino CI: PR ${CHANGE_ID} ${currentBuild.result}!",
|
||||
body: """
|
||||
Status: ${currentBuild.result}
|
||||
Pull Request Title: ${CHANGE_TITLE}
|
||||
Pull Request: ${CHANGE_URL}
|
||||
Branch: ${CHANGE_BRANCH}
|
||||
Commit Hash: ${gitPrInfo.commitSubject}
|
||||
Commit Subject: ${gitPrInfo.commitHash}
|
||||
Jenkins Build: ${RUN_DISPLAY_URL}
|
||||
""",
|
||||
to: "${notifyPeople}"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def gitSubmoduleUpdate(String repository_name) {
|
||||
dir ("${WORKDIR}/${repository_name}") {
|
||||
sh label: "Init ${repository_name} submodules",
|
||||
script:
|
||||
"""
|
||||
git submodule init && git submodule update \
|
||||
--init \
|
||||
--no-fetch \
|
||||
--recursive
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
def buildDockerImage() {
|
||||
sh """
|
||||
docker build --tag=${DOCKER_IMAGE_TAG} --file=.ci/openvino-onnx/Dockerfile \
|
||||
--build-arg http_proxy=http://proxy-chain.intel.com:911/ \
|
||||
--build-arg https_proxy=http://proxy-chain.intel.com:912/ .
|
||||
"""
|
||||
}
|
||||
|
||||
def runTests() {
|
||||
sh """
|
||||
docker run --name ${DOCKER_CONTAINER_NAME} \
|
||||
--volume ${HOME}/ONNX_CI/onnx-models-28-Oct/.onnx/model_zoo:/root/.onnx/model_zoo \
|
||||
--volume ${HOME}/ONNX_CI/onnx-models/.onnx/model_zoo/MSFT:/root/.onnx/model_zoo/MSFT \
|
||||
${DOCKER_IMAGE_TAG}
|
||||
"""
|
||||
}
|
||||
|
||||
pipeline {
|
||||
agent {
|
||||
label "OpenVino"
|
||||
}
|
||||
environment {
|
||||
PROJECT_NAME = "openvino"
|
||||
WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
|
||||
}
|
||||
options {
|
||||
skipDefaultCheckout true
|
||||
timeout(activity: true, time: 10, unit: 'MINUTES')
|
||||
}
|
||||
stages {
|
||||
stage("Clone repository") {
|
||||
steps{
|
||||
stopPreviousRunningBuilds()
|
||||
dir("${WORKDIR}") {
|
||||
checkout scm
|
||||
}
|
||||
gitSubmoduleUpdate(PROJECT_NAME)
|
||||
}
|
||||
}
|
||||
stage("Prepare Docker environment") {
|
||||
steps{
|
||||
dir("${WORKDIR}") {
|
||||
buildDockerImage()
|
||||
}
|
||||
}
|
||||
}
|
||||
stage("Run tests") {
|
||||
options {
|
||||
timeout(time: 15, unit: 'MINUTES')
|
||||
}
|
||||
steps{
|
||||
runTests()
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
failure {
|
||||
script {
|
||||
gitPrInfo = getGitPrInfo(PROJECT_NAME)
|
||||
notifyByEmail(gitPrInfo)
|
||||
}
|
||||
}
|
||||
cleanup {
|
||||
dir("${WORKDIR}") {
|
||||
deleteDir()
|
||||
sh """
|
||||
docker image prune -f
|
||||
docker rm -f ${DOCKER_CONTAINER_NAME}
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
// Copyright (C) 2018-2020 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
timeout(30)
|
||||
{
|
||||
node(LABEL) {
|
||||
|
||||
BUILD_WORKSPACE = "$WORKSPACE/$BUILD_NUMBER"
|
||||
WATCHDOG_ROOT = "$BUILD_WORKSPACE/.ci/openvino-onnx/watchdog"
|
||||
VENV_PATH = "${BUILD_WORKSPACE}/.wdvenv"
|
||||
|
||||
try {
|
||||
stage("Clone repository") {
|
||||
dir ("$BUILD_WORKSPACE") {
|
||||
checkout([$class: 'GitSCM', branches: [[name: "*/$BRANCH"]],
|
||||
doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]], submoduleCfg: [],
|
||||
userRemoteConfigs: [[credentialsId: "${GITHUB_KEY}", url: "${OPEN_VINO_URL}"]]])
|
||||
}
|
||||
}
|
||||
stage("Prepare environment") {
|
||||
sh """#!/bin/bash
|
||||
if [ ! -d ${VENV_PATH} ]; then
|
||||
python3 -m venv ${VENV_PATH}
|
||||
source ${VENV_PATH}/bin/activate
|
||||
pip install -r ${WATCHDOG_ROOT}/requirements.txt
|
||||
fi
|
||||
"""
|
||||
}
|
||||
stage("Run script") {
|
||||
withCredentials([
|
||||
usernamePassword(credentialsId: '7157091e-bc04-42f0-99fd-dc4da2922a55',
|
||||
usernameVariable: 'username',
|
||||
passwordVariable: 'password')])
|
||||
{
|
||||
dir ("$BUILD_WORKSPACE") {
|
||||
sh """#!/bin/bash
|
||||
source ${VENV_PATH}/bin/activate
|
||||
export PYTHONHTTPSVERIFY=0
|
||||
python ${WATCHDOG_ROOT}/src/main.py \
|
||||
--msteams-url=${MSTEAMS_URL_FILE} \
|
||||
--github-credentials '${username}' '${password}' \
|
||||
--github-org=${GITHUB_ORG} \
|
||||
--github-project=${GITHUB_PROJECT} \
|
||||
--jenkins-token=${JENKINS_TOKEN_FILE} \
|
||||
--jenkins-server=${JENKINS_SERVER} \
|
||||
--jenkins-user=${JENKINS_USER} \
|
||||
--ci-job=${CI_JOB_NAME} \
|
||||
--watchdog-job=${WATCHDOG_JOB_NAME}
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
echo "$e"
|
||||
currentBuild.result = "FAILURE"
|
||||
} finally {
|
||||
stage("Cleanup") {
|
||||
sh """
|
||||
cd $BUILD_WORKSPACE
|
||||
rm -rf ..?* .[!.]* *
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
python-jenkins==1.7.0
|
||||
retrying==1.3.3
|
||||
pygithub==1.51
|
||||
timeout-decorator==0.4.1
|
||||
requests==2.23.0
|
||||
wheel
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (C) 2018-2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
import timeout_decorator
|
||||
from datetime import datetime
|
||||
from retrying import retry
|
||||
from github import Github, GithubException
|
||||
|
||||
# Logging
|
||||
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
_RETRY_LIMIT = 3
|
||||
_RETRY_COOLDOWN_MS = 2000
|
||||
_REQUEST_TIMEOUT_S = 10
|
||||
|
||||
|
||||
class GitWrapper:
|
||||
"""Class wrapping PyGithub API.
|
||||
|
||||
The purpose of this class is to wrap methods from PyGithub API used in Watchdog, for less error-prone and
|
||||
more convenient use. Docs for used API, including wrapped methods can be found at:
|
||||
https://pygithub.readthedocs.io/en/latest/introduction.html
|
||||
|
||||
:param github_credentials: Credentials used for GitHub
|
||||
:param repository: GitHub repository name
|
||||
:param project: GitHub project name
|
||||
:type github_credentials: String
|
||||
:type repository: String
|
||||
:type project: String
|
||||
"""
|
||||
|
||||
def __init__(self, github_credentials, repository, project):
|
||||
self.git = Github(*github_credentials)
|
||||
self.repository = repository
|
||||
self.project = project
|
||||
self.github_credentials = github_credentials
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_git_time(self):
|
||||
"""Retrieve time from GitHub.
|
||||
|
||||
Used to reliably determine time during Watchdog run.
|
||||
|
||||
:return: Datetime object describing current time
|
||||
:rtype: datetime
|
||||
"""
|
||||
try:
|
||||
datetime_object = self._get_git_time()
|
||||
except ValueError as e:
|
||||
raise GitWrapperError(str(e))
|
||||
except GithubException as e:
|
||||
message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
|
||||
raise GitWrapperError(message)
|
||||
except timeout_decorator.TimeoutError:
|
||||
message = 'GitHub Exception during API status retrieval. Timeout during API request.'
|
||||
raise GitWrapperError(message)
|
||||
return datetime_object
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_pull_requests(self):
|
||||
"""Retrieve paginated list of pull requests from GitHub.
|
||||
|
||||
:return: Paginated list of Pull Requests in GitHub repo
|
||||
:rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
|
||||
"""
|
||||
try:
|
||||
prs = self._get_pull_requests()
|
||||
except GithubException as e:
|
||||
message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
|
||||
raise GitWrapperError(message)
|
||||
return prs
|
||||
|
||||
@timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
|
||||
def _get_git_time(self):
|
||||
"""Private method retrieving time from GitHub.
|
||||
|
||||
:return: Datetime object describing current time
|
||||
:rtype: datetime
|
||||
"""
|
||||
datetime_string = self.git.get_api_status().raw_headers.get('date', '')
|
||||
datetime_format = '%a, %d %b %Y %H:%M:%S %Z'
|
||||
datetime_object = datetime.strptime(datetime_string, datetime_format)
|
||||
return datetime_object
|
||||
|
||||
@timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
|
||||
def _get_pull_requests(self):
|
||||
"""Private method retrieving pull requests from GitHub.
|
||||
|
||||
:return: Paginated list of Pull Requests in GitHub repo
|
||||
:rtype: github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
|
||||
"""
|
||||
return self.git.get_organization(self.repository).get_repo(self.project).get_pulls()
|
||||
|
||||
|
||||
class GitWrapperError(Exception):
|
||||
"""Base class for exceptions raised in GitWrapper.
|
||||
|
||||
:param message Explanation of the error
|
||||
"""
|
||||
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
log.exception(message)
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (C) 2018-2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import requests
|
||||
import jenkins
|
||||
import logging
|
||||
from retrying import retry
|
||||
|
||||
# Logging
|
||||
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
_RETRY_LIMIT = 3
|
||||
_RETRY_COOLDOWN_MS = 5000
|
||||
|
||||
|
||||
class JenkinsWrapper:
|
||||
"""Class wrapping Python-Jenkins API.
|
||||
|
||||
The purpose of this class is to wrap methods from Python-Jenkins API used in Watchdog, for less error-prone and
|
||||
more convenient use. Docs for used API, including wrapped methods can be found at:
|
||||
https://python-jenkins.readthedocs.io/en/latest/
|
||||
|
||||
:param jenkins_token: Token used for Jenkins
|
||||
:param jenkins_user: Username used to connect to Jenkins
|
||||
:param jenkins_server: Jenkins server address
|
||||
:type jenkins_token: String
|
||||
:type jenkins_user: String
|
||||
:type jenkins_server: String
|
||||
"""
|
||||
|
||||
def __init__(self, jenkins_token, jenkins_user, jenkins_server):
|
||||
self.jenkins_server = jenkins_server
|
||||
self.jenkins = jenkins.Jenkins(jenkins_server, username=jenkins_user,
|
||||
password=jenkins_token)
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_build_console_output(self, job_name, build_number):
|
||||
return self.jenkins.get_build_console_output(job_name, build_number)
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_job_info(self, job_name):
|
||||
return self.jenkins.get_job_info(job_name)
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_build_info(self, job_name, build_number):
|
||||
return self.jenkins.get_build_info(job_name, build_number)
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_queue_item(self, queue_id):
|
||||
"""Attempt to retrieve Jenkins job queue item.
|
||||
|
||||
Exception communicating queue doesn't exist is expected,
|
||||
in that case method returns empty dict.
|
||||
|
||||
:param queue_id: Jenkins job queue ID number
|
||||
:type queue_id: int
|
||||
:return: Dictionary representing Jenkins job queue item
|
||||
:rtype: dict
|
||||
"""
|
||||
try:
|
||||
return self.jenkins.get_queue_item(queue_id)
|
||||
except Exception as e:
|
||||
# Exception 'queue does not exist' is expected behaviour when job is running
|
||||
if 'queue' in str(e) and 'does not exist' in str(e):
|
||||
return {}
|
||||
else:
|
||||
raise
|
||||
|
||||
@retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
|
||||
def get_idle_ci_hosts(self):
|
||||
"""Query Jenkins for idle servers.
|
||||
|
||||
Send GET request to Jenkins server, querying for idle servers labeled
|
||||
for OpenVino-ONNX CI job.
|
||||
|
||||
:return: Number of idle hosts delegated to OpenVino-ONNX CI
|
||||
:rtype: int
|
||||
"""
|
||||
jenkins_request_url = self.jenkins_server + 'label/ci&&onnx/api/json?pretty=true'
|
||||
try:
|
||||
log.info('Sending request to Jenkins: %s', jenkins_request_url)
|
||||
r = requests.Request(method='GET', url=jenkins_request_url, verify=False)
|
||||
response = self.jenkins.jenkins_request(r).json()
|
||||
return int(response['totalExecutors']) - int(response['busyExecutors'])
|
||||
except Exception as e:
|
||||
log.exception('Failed to send request to Jenkins!\nException message: %s', str(e))
|
||||
raise
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (C) 2018-2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from watchdog import Watchdog
|
||||
|
||||
DEFAULT_MSTEAMS_URL_FILE = '/home/lab_nerval/tokens/msteams_url'
|
||||
DEFAULT_GITHUB_ORGANIZATION = 'openvinotoolkit'
|
||||
DEFAULT_GITHUB_PROJECT = 'openvino'
|
||||
DEFAULT_JENKINS_TOKEN_FILE = '/home/lab_nerval/tokens/crackerjack'
|
||||
DEFAULT_JENKINS_SERVER = 'https://crackerjack.intel.com/'
|
||||
DEFAULT_JENKINS_USER = 'lab_nerval'
|
||||
DEFAULT_CI_JOB_NAME = 'onnx/OpenVino_CI'
|
||||
DEFAULT_WATCHDOG_JOB_NAME = 'onnx/ci_watchdog'
|
||||
|
||||
|
||||
def main(args):
|
||||
"""
|
||||
Read args passed to script, load tokens and run watchdog.
|
||||
|
||||
Keyword arguments:
|
||||
:param args: arguments parsed by argparse ArgumentParser
|
||||
|
||||
:return: returns status code 0 on successful completion
|
||||
|
||||
"""
|
||||
jenkins_server = args.jenkins_server.strip()
|
||||
jenkins_user = args.jenkins_user.strip()
|
||||
jenkins_token = open(args.jenkins_token).read().replace('\n', '').strip()
|
||||
msteams_url = open(args.msteams_url).read().replace('\n', '').strip()
|
||||
github_credentials = args.github_credentials
|
||||
github_org = args.github_org
|
||||
github_project = args.github_project
|
||||
ci_job = args.ci_job.strip()
|
||||
watchdog_job = args.watchdog_job.strip()
|
||||
quiet = args.quiet
|
||||
|
||||
wd = Watchdog(jenkins_token=jenkins_token,
|
||||
jenkins_server=jenkins_server,
|
||||
jenkins_user=jenkins_user,
|
||||
github_credentials=github_credentials,
|
||||
git_org=github_org,
|
||||
git_project=github_project,
|
||||
msteams_url=msteams_url,
|
||||
ci_job_name=ci_job,
|
||||
watchdog_job_name=watchdog_job)
|
||||
wd.run(quiet=quiet)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument('--msteams-url', help='Path to MS Teams channel url to communicate messages.',
|
||||
default=DEFAULT_MSTEAMS_URL_FILE, action='store', required=False)
|
||||
|
||||
parser.add_argument('--github-credentials', help='GitHub user credentials to access repo.',
|
||||
nargs="+", required=True)
|
||||
|
||||
parser.add_argument('--github-org', help='Name of organization on GitHub.',
|
||||
default=DEFAULT_GITHUB_ORGANIZATION, action='store', required=False)
|
||||
|
||||
parser.add_argument('--github-project', help='Name of project on GitHub.',
|
||||
default=DEFAULT_GITHUB_PROJECT, action='store', required=False)
|
||||
|
||||
parser.add_argument('--jenkins-token', help='Path to Jenkins user token to access build info.',
|
||||
default=DEFAULT_JENKINS_TOKEN_FILE, action='store', required=False)
|
||||
|
||||
parser.add_argument('--jenkins-server', help='Jenkins server address.',
|
||||
default=DEFAULT_JENKINS_SERVER, action='store', required=False)
|
||||
|
||||
parser.add_argument('--jenkins-user', help='Jenkins user used to log in.',
|
||||
default=DEFAULT_JENKINS_USER, action='store', required=False)
|
||||
|
||||
parser.add_argument('--ci-job', help='Jenkins CI job name.',
|
||||
default=DEFAULT_CI_JOB_NAME, action='store', required=False)
|
||||
|
||||
parser.add_argument('--watchdog-job', help='Jenkins CI Watchdog job name.',
|
||||
default=DEFAULT_WATCHDOG_JOB_NAME, action='store', required=False)
|
||||
|
||||
parser.add_argument('--quiet', help="Quiet mode - doesn\'t send message to communicator.",
|
||||
action='store_true', required=False)
|
||||
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(args))
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (C) 2018-2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class MSTeamsCommunicator:
|
||||
"""Class communicating with MSTeams using Incoming Webhook.
|
||||
|
||||
The purpose of this class is to use MSTeams API to send message.
|
||||
Docs for used API, including wrapped methods can be found at:
|
||||
https://docs.microsoft.com/en-us/outlook/actionable-messages/send-via-connectors
|
||||
"""
|
||||
|
||||
def __init__(self, _ci_alerts_channel_url):
|
||||
self._ci_alerts_channel_url = _ci_alerts_channel_url
|
||||
self._queued_messages = {
|
||||
self._ci_alerts_channel_url: [],
|
||||
}
|
||||
|
||||
@property
|
||||
def messages(self):
|
||||
"""
|
||||
Get list of queued messages.
|
||||
|
||||
:return: List of queued messages
|
||||
:return type: List[String]
|
||||
"""
|
||||
return self._queued_messages.values()
|
||||
|
||||
def queue_message(self, message):
|
||||
"""
|
||||
Queue message to be sent later.
|
||||
|
||||
:param message: Message content
|
||||
:type message: String
|
||||
"""
|
||||
self._queued_messages[self._ci_alerts_channel_url].append(message)
|
||||
|
||||
def _parse_text(self, watchdog_log, message):
|
||||
"""
|
||||
Parse text to display as alert.
|
||||
|
||||
:param watchdog_log: Watchdog log content
|
||||
:param message: Unparsed message content
|
||||
:type watchdog_log: String
|
||||
:type message: String
|
||||
"""
|
||||
message_split = message.split('\n')
|
||||
log_url = None
|
||||
if len(message_split) == 3:
|
||||
log_url = message_split[-1]
|
||||
title = message_split[0]
|
||||
text = message_split[1]
|
||||
header = watchdog_log.split(' - ')
|
||||
header_formatted = '{} - [Watchdog Log]({})'.format(header[0], header[1])
|
||||
return title, log_url, '{}\n\n{}'.format(header_formatted, text)
|
||||
|
||||
def _json_request_content(self, title, log_url, text_formatted):
|
||||
"""
|
||||
Create final json request to send message to MS Teams channel.
|
||||
|
||||
:param title: Title of alert
|
||||
:param log_url: URL to PR
|
||||
:param text_formatted: General content of alert - finally formatted
|
||||
:type title: String
|
||||
:type title: String
|
||||
:type title: String
|
||||
"""
|
||||
data = {
|
||||
'@context': 'https://schema.org/extensions',
|
||||
'@type': 'MessageCard',
|
||||
'themeColor': '0072C6',
|
||||
'title': title,
|
||||
'text': text_formatted,
|
||||
'potentialAction':
|
||||
[
|
||||
{
|
||||
'@type': 'OpenUri',
|
||||
'name': 'Open PR',
|
||||
'targets':
|
||||
[
|
||||
{
|
||||
'os': 'default',
|
||||
'uri': log_url,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
return data
|
||||
|
||||
def _send_to_channel(self, watchdog_log, message_queue, channel_url):
|
||||
"""
|
||||
Send MSTeams message to specified channel.
|
||||
|
||||
:param watchdog_log: Watchdog log content
|
||||
:param message_queue: Queued messages to send
|
||||
:param channel_url: Channel url
|
||||
:type watchdog_log: String
|
||||
:type message_queue: String
|
||||
:type channel_url: String
|
||||
|
||||
"""
|
||||
for message in message_queue:
|
||||
title, log_url, text_formatted = self._parse_text(watchdog_log, message)
|
||||
data = self._json_request_content(title, log_url, text_formatted)
|
||||
|
||||
try:
|
||||
requests.post(url=channel_url, json=data)
|
||||
except Exception as ex:
|
||||
raise Exception('!!CRITICAL!! MSTeamsCommunicator: Could not send message '
|
||||
'due to {}'.format(ex))
|
||||
|
||||
def send_message(self, watchdog_log, quiet=False):
|
||||
"""
|
||||
Send queued messages as single communication.
|
||||
|
||||
:param watchdog_log: Watchdog log content
|
||||
:param quiet: Flag for disabling sending report through MS Teams
|
||||
:type watchdog_log: String
|
||||
:type quiet: Boolean
|
||||
"""
|
||||
for channel, message_queue in self._queued_messages.items():
|
||||
if not quiet and message_queue:
|
||||
self._send_to_channel(watchdog_log, message_queue, channel)
|
||||
|
|
@ -0,0 +1,505 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
# Copyright (C) 2018-2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import datetime
|
||||
import time
|
||||
import re
|
||||
import logging
|
||||
import requests
|
||||
from ms_teams_communicator import MSTeamsCommunicator
|
||||
from jenkins_wrapper import JenkinsWrapper
|
||||
from jenkins import NotFoundException
|
||||
from git_wrapper import GitWrapper, GitWrapperError
|
||||
import os
|
||||
import json
|
||||
|
||||
# Logging
|
||||
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
# Watchdog static constant variables
|
||||
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
_BUILD_DURATION_THRESHOLD = datetime.timedelta(minutes=60)
|
||||
_CI_START_THRESHOLD = datetime.timedelta(minutes=30)
|
||||
_AWAITING_JENKINS_THRESHOLD = datetime.timedelta(minutes=5)
|
||||
_WATCHDOG_DIR = os.path.expanduser('~')
|
||||
_PR_REPORTS_CONFIG_KEY = 'pr_reports'
|
||||
_CI_BUILD_FAIL_MESSAGE = 'ERROR: py3: commands failed'
|
||||
_CI_BUILD_SUCCESS_MESSAGE = 'py3: commands succeeded'
|
||||
_GITHUB_CI_CHECK_NAME = 'OpenVINO-ONNX'
|
||||
|
||||
INTERNAL_ERROR_MESSAGE_HEADER = '!!! --- !!! INTERNAL WATCHDOG ERROR !!! --- !!!'
|
||||
ERROR_MESSAGE_HEADER = '!!! OpenVino-ONNX CI Error !!!'
|
||||
WARNING_MESSAGE_HEADER = 'OpenVino-ONNX CI WARNING'
|
||||
INFO_MESSAGE_HEADER = 'OpenVino-ONNX CI INFO'
|
||||
|
||||
|
||||
class Watchdog:
|
||||
"""Class describing OpenVino-ONNX-CI Watchdog.
|
||||
|
||||
Watchdog connects to GitHub and retrieves the list of current pull requests (PRs) in
|
||||
OpenVino repository. Then it connects to specified Jenkins server to
|
||||
check CI jobs associated with every PR. Watchdog verifies time durations for Jenkins
|
||||
initial response, job queue and execution against time treshold constants. Every fail
|
||||
is logged and reported through MS Teams communicators.
|
||||
|
||||
:param jenkins_token: Token used for Jenkins
|
||||
:param jenkins_server: Jenkins server address
|
||||
:param jenkins_user: Username used to connect to Jenkins
|
||||
:param github_credentials: Credentials used to connect to GitHub
|
||||
:param msteams_url: URL used to connect to MS Teams channel
|
||||
:param ci_job_name: OpenVino-ONNX CI job name used in Jenkins
|
||||
:param watchdog_job_name: Watchdog job name used in Jenkins
|
||||
:type jenkins_token: String
|
||||
:type jenkins_server: String
|
||||
:type jenkins_user: String
|
||||
:type github_credentials: String
|
||||
:type msteams_url: String
|
||||
:type ci_job_name: String
|
||||
:type watchdog_job_name: String
|
||||
|
||||
.. note::
|
||||
Watchdog and OpenVino-ONNX CI job must be placed on the same Jenkins server.
|
||||
"""
|
||||
|
||||
def __init__(self, jenkins_token, jenkins_server, jenkins_user, github_credentials, git_org,
|
||||
git_project, msteams_url, ci_job_name, watchdog_job_name):
|
||||
self._config_path = os.path.join(_WATCHDOG_DIR, '{}/.{}_ci_watchdog.json'.format(_WATCHDOG_DIR, git_project))
|
||||
# Jenkins Wrapper object for CI job
|
||||
self._jenkins = JenkinsWrapper(jenkins_token,
|
||||
jenkins_user=jenkins_user,
|
||||
jenkins_server=jenkins_server)
|
||||
# Load GitHub token and log in, retrieve pull requests
|
||||
self._git = GitWrapper(github_credentials, repository=git_org, project=git_project)
|
||||
# Create MS Teams api object
|
||||
self._msteams_hook = MSTeamsCommunicator(msteams_url)
|
||||
self._ci_job_name = ci_job_name.lower()
|
||||
self._watchdog_job_name = watchdog_job_name
|
||||
# Read config file
|
||||
self._config = self._read_config_file()
|
||||
# Time at Watchdog initiation
|
||||
self._now_time = datetime.datetime.now()
|
||||
self._current_prs = {}
|
||||
self._ms_teams_enabled = True
|
||||
|
||||
def run(self, quiet=False):
|
||||
"""Run main watchdog logic.
|
||||
|
||||
Retrieve list of pull requests and pass it to the method responsible for checking them.
|
||||
|
||||
:param quiet: Flag for disabling sending report through communicator
|
||||
:type quiet: Boolean
|
||||
"""
|
||||
try:
|
||||
pull_requests = self._git.get_pull_requests()
|
||||
except GitWrapperError:
|
||||
message = 'Failed to retrieve Pull Requests!'
|
||||
log.exception(message)
|
||||
self._queue_message(message, message_severity='internal')
|
||||
# Check all pull requests
|
||||
for pr in pull_requests:
|
||||
try:
|
||||
self._check_pr(pr)
|
||||
except Exception as e:
|
||||
log.exception(str(e))
|
||||
self._queue_message(str(e), message_severity='internal', pr=pr)
|
||||
self._update_config()
|
||||
self._send_message(quiet=quiet)
|
||||
|
||||
def _read_config_file(self):
|
||||
"""Read Watchdog config file stored on the system.
|
||||
|
||||
The file stores every fail already reported along with timestamp. This
|
||||
mechanism is used to prevent Watchdog from reporting same failure
|
||||
multiple times. In case there's no config under the expected path,
|
||||
appropriate data structure is created and returned.
|
||||
|
||||
:return: Returns dict of dicts with reported fails with their timestamps
|
||||
:rtype: dict of dicts
|
||||
"""
|
||||
if os.path.isfile(self._config_path):
|
||||
log.info('Reading config file in: {}'.format(self._config_path))
|
||||
file = open(self._config_path, 'r')
|
||||
data = json.load(file)
|
||||
else:
|
||||
log.info('No config file found in: {}'.format(self._config_path))
|
||||
data = {_PR_REPORTS_CONFIG_KEY: {}}
|
||||
return data
|
||||
|
||||
def _check_pr(self, pr):
|
||||
"""Check pull request (if there's no reason to skip).
|
||||
|
||||
Retrieve list of statuses for every PR's last commit and interpret them. Filters out statuses
|
||||
unrelated to OpenVino-ONNX Jenkins CI and passes relevant statuses to method that interprets them.
|
||||
If no commit statuses related to Jenkins are available after time defined by
|
||||
**_AWAITING_JENKINS_THRESHOLD** calls appropriate method to check for builds waiting in queue.
|
||||
|
||||
:param pr: GitHub Pull Requests
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
"""
|
||||
log.info('===============================================')
|
||||
log.info('Checking PR#{}'.format(pr.number))
|
||||
# Get last Jenkins status
|
||||
last_status = self._get_last_status(pr)
|
||||
# Append PR checked in current run for Watchdog config
|
||||
self._current_prs[str(pr.number)] = self._get_pr_timestamps(pr, last_status)
|
||||
if self._should_ignore(pr) or self._updated_since_last_run(pr):
|
||||
log.info('Ignoring PR#{}'.format(pr.number))
|
||||
|
||||
return
|
||||
|
||||
# Calculate time passed since PR update (any commit, merge or comment)
|
||||
pr_time_delta = self._now_time - pr.updated_at
|
||||
if last_status:
|
||||
# Interpret found CI statuses
|
||||
log.info('Last status: {} at {}'.format(last_status.description, last_status.updated_at))
|
||||
self._interpret_status(last_status, pr)
|
||||
elif pr_time_delta > _CI_START_THRESHOLD:
|
||||
# If there's no status after assumed time - check if build is waiting in queue
|
||||
log.info('CI for PR {}: NO JENKINS STATUS YET'.format(pr.number))
|
||||
self._check_missing_status(pr)
|
||||
|
||||
@staticmethod
|
||||
def _get_pr_timestamps(pr, last_status):
|
||||
"""Get dict containing PR timestamp and last status timestamp.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
|
||||
:return: Dictionary with PR and last status update timestamps
|
||||
:rtype: dict
|
||||
"""
|
||||
pr_timestamp = time.mktime(pr.updated_at.timetuple())
|
||||
if last_status:
|
||||
status_timestamp = time.mktime(last_status.updated_at.timetuple())
|
||||
else:
|
||||
status_timestamp = None
|
||||
pr_dict = {'pr_timestamp': pr_timestamp,
|
||||
'status_timestamp': status_timestamp}
|
||||
return pr_dict
|
||||
|
||||
@staticmethod
|
||||
def _get_last_status(pr):
|
||||
"""Get last commit status posted from Jenkins.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
|
||||
:return: Either last PR status posted from Jenkins or None
|
||||
:rtype: github.CommitStatus.CommitStatus
|
||||
"""
|
||||
# Find last commit in PR
|
||||
last_commit = pr.get_commits().reversed[0]
|
||||
# Get statuses and filter them to contain only those related to Jenkins CI
|
||||
# and check if CI in Jenkins started
|
||||
statuses = last_commit.get_statuses()
|
||||
jenk_statuses = [stat for stat in statuses if
|
||||
_GITHUB_CI_CHECK_NAME in stat.context]
|
||||
try:
|
||||
last_status = jenk_statuses[0]
|
||||
except IndexError:
|
||||
last_status = None
|
||||
return last_status
|
||||
|
||||
@staticmethod
|
||||
def _should_ignore(pr):
|
||||
"""Determine if PR should be ignored.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
|
||||
:return: Returns True if PR should be ignored
|
||||
:rtype: Bool
|
||||
"""
|
||||
# Ignore PR if it has WIP label or WIP in title
|
||||
if 'WIP' in pr.title:
|
||||
log.info('PR#{} should be ignored. WIP tag in title.'.format(pr.number))
|
||||
return True
|
||||
|
||||
label_names = [label.name for label in pr.labels]
|
||||
if 'WIP' in label_names:
|
||||
log.info('PR#{} should be ignored. WIP label present.'.format(pr.number))
|
||||
return True
|
||||
|
||||
# Ignore PR if base ref is not master
|
||||
if 'master' not in pr.base.ref:
|
||||
log.info('PR#{} should be ignored. Base ref is not master'.format(pr.number))
|
||||
return True
|
||||
|
||||
# Ignore PR if mergeable state is 'dirty' or 'behind'.
|
||||
# Practically this ignores PR in case of merge conflicts
|
||||
ignored_mergeable_states = ['behind', 'dirty', 'draft']
|
||||
if pr.mergeable_state in ignored_mergeable_states:
|
||||
log.info('PR#{} should be ignored. Mergeable state is {}. '.format(pr.number, pr.mergeable_state))
|
||||
return True
|
||||
|
||||
# If no criteria for ignoring PR are met - return false
|
||||
return False
|
||||
|
||||
def _updated_since_last_run(self, pr):
|
||||
# Ignore if PR was already checked and there was no update in meantime
|
||||
pr_number = str(pr.number)
|
||||
current_pr_timestamps = self._current_prs.get(pr_number)
|
||||
last_pr_timestamps = self._config[_PR_REPORTS_CONFIG_KEY].get(pr_number)
|
||||
if current_pr_timestamps == last_pr_timestamps:
|
||||
log.info('PR#{} - No update since last check'.format(pr.number))
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _check_missing_status(self, pr):
|
||||
"""Verify if missing status is expected.
|
||||
|
||||
This method checks if CI build for last was scheduled and still waits in queue for
|
||||
executor.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
"""
|
||||
pr_time_delta = self._now_time - pr.updated_at
|
||||
try:
|
||||
build_number = self._build_scheduled(pr)
|
||||
if self._build_in_queue(pr, build_number):
|
||||
message = ('PR# {}: build waiting in queue after {} minutes.'
|
||||
.format(pr.number, pr_time_delta.seconds / 60))
|
||||
severity = 'warning'
|
||||
else:
|
||||
message = ('PR# {}: missing status on GitHub after {} minutes.'
|
||||
.format(pr.number, pr_time_delta.seconds / 60))
|
||||
severity = 'error'
|
||||
self._queue_message(message, message_severity=severity, pr=pr)
|
||||
except TypeError:
|
||||
log.info('Committer outside of OpenVino organization')
|
||||
|
||||
def _build_scheduled(self, pr):
|
||||
"""Check if Jenkins build corresponding to PR was scheduled.
|
||||
|
||||
This method takes last Jenkins build for given PR and compares hash from Jenkins console output
|
||||
and sha from PR object to determine if CI build for appropriate commit was scheduled.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
|
||||
:return: Returns build number or -1 if no build found
|
||||
:rtype: int
|
||||
"""
|
||||
pr_number = str(pr.number)
|
||||
project_name_full = self._ci_job_name + '/PR-' + pr_number
|
||||
|
||||
try:
|
||||
# Retrieve console output from last Jenkins build for job corresponding to this PR
|
||||
last_build_number = self._jenkins.get_job_info(project_name_full)['lastBuild']['number']
|
||||
console_output = self._jenkins.get_build_console_output(project_name_full, last_build_number)
|
||||
# Check if CI build was scheduled - commit hash on GH must match hash in last Jenkins build console output
|
||||
# Retrieve hash from Jenkins output
|
||||
match_string = '(?:Obtained .ci/[a-zA-Z/]+Jenkinsfile from ([a-z0-9]{40}))'
|
||||
retrieved_sha = re.search(match_string, console_output).group(1)
|
||||
if retrieved_sha == pr.get_commits().reversed[0].sha:
|
||||
return last_build_number
|
||||
else:
|
||||
return -1
|
||||
except (NotFoundException, AttributeError, requests.exceptions.HTTPError):
|
||||
message = ('PR #{}: Jenkins build corresponding to commit {} not found!'
|
||||
.format(pr_number, pr.get_commits().reversed[0].sha))
|
||||
self._queue_message(message, message_severity='error', pr=pr)
|
||||
return -1
|
||||
|
||||
def _build_in_queue(self, pr, build_number):
|
||||
"""Check if Jenkins build waits in queue.
|
||||
|
||||
This method verifies if CI build is waiting in queue based on console output.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:param build_number: Jenkins build number to retrieve console output from
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
:type build_number: int
|
||||
|
||||
:return: Returns True if CI build is waiting in queue
|
||||
:rtype: Bool
|
||||
"""
|
||||
pr_number = str(pr.number)
|
||||
project_name_full = self._ci_job_name + '/PR-' + pr_number
|
||||
# Retrieve console output
|
||||
try:
|
||||
console_output = self._jenkins.get_build_console_output(project_name_full, build_number)
|
||||
except NotFoundException:
|
||||
return False
|
||||
# Check if build is waiting in queue (and not already running on an executor)
|
||||
if 'Waiting for next available executor on' in console_output \
|
||||
and 'Running on' not in console_output:
|
||||
log.info('CI for PR %s: WAITING IN QUEUE', pr_number)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _interpret_status(self, status, pr):
|
||||
"""
|
||||
Verify GitHub status passed to the method.
|
||||
|
||||
This method verifies last commit status for given PR, calling appropriate methods
|
||||
to further validate the status.
|
||||
|
||||
:param status: GitHub commit status
|
||||
:param pr: Single PR being currently checked
|
||||
:type status: github.CommitStatus.CommitStatus
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
"""
|
||||
try:
|
||||
# Retrieve build number for Jenkins build related to this PR
|
||||
build_number = self._retrieve_build_number(status.target_url)
|
||||
# CI build finished - verify if expected output is present
|
||||
finished_statuses = ['Build finished', 'This commit cannot be built', 'This commit looks good']
|
||||
pending_statuses = ['This commit is being built', 'Testing in progress',
|
||||
'This commit is scheduled to be built']
|
||||
if any(phrase in status.description for phrase in finished_statuses):
|
||||
self._check_finished(pr, build_number)
|
||||
# CI build in progress - verify timeouts for build queue and duration
|
||||
elif any(phrase in status.description for phrase in pending_statuses):
|
||||
self._check_in_progress(pr, build_number)
|
||||
else:
|
||||
message = 'ONNX CI job for PR# {}: unrecognized status: {}'.format(pr.number, status.description)
|
||||
self._queue_message(message, message_severity='error', pr=pr)
|
||||
except Exception:
|
||||
# Log Watchdog internal error in case any status can't be properly verified
|
||||
message = 'Failed to verify status "{}" for PR# {}'.format(status.description, pr.number)
|
||||
log.exception(message)
|
||||
self._queue_message(message, message_severity='internal', pr=pr)
|
||||
|
||||
def _retrieve_build_number(self, url):
|
||||
"""Retrieve Jenkins CI job build number from URL address coming from GitHub commit status.
|
||||
|
||||
:param url: URL address from GitHub commit status
|
||||
:type url: String
|
||||
|
||||
:return: Returns build number
|
||||
:rtype: int
|
||||
"""
|
||||
# Retrieve the build number from url string
|
||||
match_obj = re.search('(?:/PR-[0-9]+/)([0-9]+)', url)
|
||||
try:
|
||||
number = int(match_obj.group(1))
|
||||
return number
|
||||
except Exception:
|
||||
log.exception('Failed to retrieve build number from url link: %s', url)
|
||||
raise
|
||||
|
||||
def _queue_message(self, message, message_severity='info', pr=None):
|
||||
"""Add a message to message queue in communicator object.
|
||||
|
||||
The queued message is constructed based on message string passed as
|
||||
a method argument and message header. Message header is mapped to message severity
|
||||
also passed as an argument.
|
||||
|
||||
:param message: Message content
|
||||
:param message_severity: Message severity level
|
||||
:type message: String
|
||||
:type message_severity: int
|
||||
"""
|
||||
log.info(message)
|
||||
internal = False
|
||||
if 'internal' in message_severity:
|
||||
message_header = INTERNAL_ERROR_MESSAGE_HEADER
|
||||
internal = True
|
||||
elif 'error' in message_severity:
|
||||
message_header = ERROR_MESSAGE_HEADER
|
||||
elif 'warning' in message_severity:
|
||||
message_header = WARNING_MESSAGE_HEADER
|
||||
else:
|
||||
message_header = INFO_MESSAGE_HEADER
|
||||
# If message is related to PR attatch url
|
||||
if pr:
|
||||
message = message + '\n' + pr.html_url
|
||||
|
||||
send = message_header + '\n' + message
|
||||
if self._ms_teams_enabled:
|
||||
self._msteams_hook.queue_message(send)
|
||||
|
||||
def _check_finished(self, pr, build_number):
|
||||
"""Verify if finished build output contains expected string for either fail or success.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:param build_number: Jenkins CI job build number
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
:type build_number: int
|
||||
"""
|
||||
pr_number = str(pr.number)
|
||||
log.info('CI for PR %s: FINISHED', pr_number)
|
||||
# Check if FINISH was valid FAIL / SUCCESS
|
||||
project_name_full = self._ci_job_name + '/PR-' + pr_number
|
||||
build_output = self._jenkins.get_build_console_output(project_name_full, build_number)
|
||||
if _CI_BUILD_FAIL_MESSAGE not in build_output \
|
||||
and _CI_BUILD_SUCCESS_MESSAGE not in build_output:
|
||||
message = ('ONNX CI job for PR #{}: finished but no tests success or fail '
|
||||
'confirmation is present in console output!'.format(pr_number))
|
||||
self._queue_message(message, message_severity='error', pr=pr)
|
||||
|
||||
def _send_message(self, quiet=False):
|
||||
"""Send messages queued in MS Teams objects to designated channel.
|
||||
|
||||
Queued messages are being sent as a single communication.
|
||||
|
||||
:param quiet: Flag for disabling sending report through communicator
|
||||
:type quiet: Boolean
|
||||
"""
|
||||
if any(messages for messages in self._msteams_hook.messages):
|
||||
try:
|
||||
watchdog_build = self._jenkins.get_job_info(self._watchdog_job_name)['lastBuild']
|
||||
watchdog_build_number = watchdog_build['number']
|
||||
watchdog_build_link = watchdog_build['url']
|
||||
except Exception:
|
||||
watchdog_build_number = 'UNKNOWN'
|
||||
watchdog_build_link = self._jenkins.jenkins_server
|
||||
send = self._watchdog_job_name + '- build ' + str(
|
||||
watchdog_build_number) + ' - ' + watchdog_build_link
|
||||
|
||||
if self._ms_teams_enabled:
|
||||
self._msteams_hook.send_message(send, quiet=quiet)
|
||||
else:
|
||||
log.info('Nothing to report.')
|
||||
|
||||
def _check_in_progress(self, pr, build_number):
|
||||
"""Check if CI build succesfully started.
|
||||
|
||||
Checks if build started within designated time threshold, and job is
|
||||
currently running - it didn't cross the time threshold.
|
||||
|
||||
:param pr: Single PR being currently checked
|
||||
:param build_number: Jenkins CI job build number
|
||||
:type pr: github.PullRequest.PullRequest
|
||||
:type build_number: int
|
||||
"""
|
||||
pr_number = str(pr.number)
|
||||
log.info('CI for PR %s: TESTING IN PROGRESS', pr_number)
|
||||
project_name_full = self._ci_job_name + '/PR-' + pr_number
|
||||
build_info = self._jenkins.get_build_info(project_name_full, build_number)
|
||||
build_datetime = datetime.datetime.fromtimestamp(build_info['timestamp'] / 1000.0)
|
||||
build_delta = self._now_time - build_datetime
|
||||
log.info('Build %s: IN PROGRESS, started: %s minutes ago', str(build_number),
|
||||
str(build_delta))
|
||||
# If build still waiting in queue
|
||||
if build_delta > _CI_START_THRESHOLD and self._build_in_queue(pr, build_number):
|
||||
message = ('ONNX CI job build #{}, for PR #{} waiting in queue after {} '
|
||||
'minutes'.format(build_number, pr_number, str(build_delta.seconds / 60)))
|
||||
self._queue_message(message, message_severity='warning', pr=pr)
|
||||
elif build_delta > _BUILD_DURATION_THRESHOLD:
|
||||
# CI job take too long, possibly froze - communicate failure
|
||||
message = ('ONNX CI job build #{}, for PR #{} started,'
|
||||
'but did not finish in designated time of {} '
|
||||
'minutes!'.format(build_number, pr_number,
|
||||
str(_BUILD_DURATION_THRESHOLD.seconds / 60)))
|
||||
self._queue_message(message, message_severity='error', pr=pr)
|
||||
|
||||
def _update_config(self):
|
||||
"""Update Watchdog config file with PRs checked in current Watchdog run, remove old entries.
|
||||
|
||||
:param current_prs: List of PR numbers checked during current Watchdog run
|
||||
:type current_prs: list of ints
|
||||
"""
|
||||
# Cleanup config of old reports
|
||||
log.info('Writing to config file at: {}'.format(self._config_path))
|
||||
new_config = {_PR_REPORTS_CONFIG_KEY: self._current_prs}
|
||||
file = open(self._config_path, 'w+')
|
||||
json.dump(new_config, file)
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
*
|
||||
!install_build_dependencies.sh
|
||||
!scripts/install_dependencies/install_openvino_dependencies.sh
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
# Set default behavior to automatically normalize line endings.
|
||||
###############################################################################
|
||||
* text=auto
|
||||
|
||||
###############################################################################
|
||||
# Set default behavior for command prompt diff.
|
||||
#
|
||||
|
|
@ -10,7 +11,9 @@
|
|||
# Note: This is only used by command line
|
||||
###############################################################################
|
||||
#*.cs diff=csharp
|
||||
|
||||
*.py text eol=lf
|
||||
|
||||
###############################################################################
|
||||
# Set the merge driver for project and solution files
|
||||
#
|
||||
|
|
@ -33,6 +36,7 @@
|
|||
#*.modelproj merge=binary
|
||||
#*.sqlproj merge=binary
|
||||
#*.wwaproj merge=binary
|
||||
|
||||
###############################################################################
|
||||
# behavior for image files
|
||||
#
|
||||
|
|
@ -41,6 +45,7 @@
|
|||
#*.jpg binary
|
||||
#*.png binary
|
||||
#*.gif binary
|
||||
|
||||
###############################################################################
|
||||
# diff behavior for common document formats
|
||||
#
|
||||
|
|
@ -58,12 +63,9 @@
|
|||
#*.PDF diff=astextplain
|
||||
#*.rtf diff=astextplain
|
||||
#*.RTF diff=astextplain
|
||||
|
||||
*.PNG filter=lfs diff=lfs merge=lfs -text
|
||||
*.png filter=lfs diff=lfs merge=lfs -text
|
||||
*.jpg filter=lfs diff=lfs merge=lfs -text
|
||||
*.gif filter=lfs diff=lfs merge=lfs -text
|
||||
*.vsdx filter=lfs diff=lfs merge=lfs -text
|
||||
*.bmp filter=lfs diff=lfs merge=lfs -text
|
||||
*.svg filter=lfs diff=lfs merge=lfs -text
|
||||
.github/scripts/workflow_rerun/tests/data/log_archive_with_error.zip filter=lfs diff=lfs merge=lfs -text
|
||||
.github/scripts/workflow_rerun/tests/data/log_archive_wo_error.zip filter=lfs diff=lfs merge=lfs -text
|
||||
|
|
|
|||
|
|
@ -1,137 +0,0 @@
|
|||
# See help here: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
|
||||
|
||||
* @openvinotoolkit/openvino-maintainers
|
||||
|
||||
# CI
|
||||
/Jenkinsfile @openvinotoolkit/openvino-ci-maintainers
|
||||
/.github/ @openvinotoolkit/openvino-ci-maintainers
|
||||
/.ci/ @openvinotoolkit/openvino-ci-maintainers
|
||||
/.github/CODEOWNERS @openvinotoolkit/openvino-admins @openvinotoolkit/openvino-maintainers
|
||||
|
||||
# Licensing:
|
||||
/licensing/ @openvinotoolkit/openvino-legal-maintainers
|
||||
/LICENSE @openvinotoolkit/openvino-legal-maintainers
|
||||
|
||||
# OpenVINO Scripts:
|
||||
/scripts/ @openvinotoolkit/openvino-scripts-maintainers
|
||||
/scripts/install_dependencies/ @openvinotoolkit/openvino-configuration-mgmt @openvinotoolkit/openvino-scripts-maintainers
|
||||
/install_build_dependencies.sh @openvinotoolkit/openvino-scripts-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
|
||||
# OpenVINO Core:
|
||||
/src/inference/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/src/common/conditional_compilation/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/src/common/itt/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/src/common/preprocessing/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/src/common/util/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/thirdparty/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/.gitmodules @openvinotoolkit/openvino-ie-maintainers
|
||||
|
||||
/src/bindings/python/ @openvinotoolkit/openvino-ie-python-api-maintainers
|
||||
/src/bindings/c/ @openvinotoolkit/openvino-c-api-maintainers
|
||||
/src/bindings/js/ @openvinotoolkit/openvino-js-api-maintainers
|
||||
/src/core/ @openvinotoolkit/openvino-ngraph-maintainers
|
||||
|
||||
# OpenVINO Transformations
|
||||
/src/common/*transformations/ @openvinotoolkit/openvino-ie-transformations-maintainers
|
||||
/src/core/src/pass/ @openvinotoolkit/openvino-ie-transformations-maintainers
|
||||
/src/core/src/pattern/ @openvinotoolkit/openvino-ie-transformations-maintainers
|
||||
/src/core/include/openvino/pass/ @openvinotoolkit/openvino-ie-transformations-maintainers
|
||||
|
||||
# OpenVINO Samples:
|
||||
/samples/c/ @openvinotoolkit/openvino-samples-maintainers @openvinotoolkit/openvino-c-api-maintainers
|
||||
/samples/cpp/ @openvinotoolkit/openvino-samples-maintainers @openvinotoolkit/openvino-maintainers
|
||||
/samples/python/ @openvinotoolkit/openvino-samples-maintainers @openvinotoolkit/openvino-ie-python-api-maintainers
|
||||
/samples/js/ @openvinotoolkit/openvino-samples-maintainers @openvinotoolkit/openvino-js-api-maintainers
|
||||
/thirdparty/zlib/ @openvinotoolkit/openvino-samples-maintainers
|
||||
/thirdparty/json/ @openvinotoolkit/openvino-samples-maintainers
|
||||
/thirdparty/gflags/ @openvinotoolkit/openvino-samples-maintainers
|
||||
/thirdparty/cnpy/ @openvinotoolkit/openvino-samples-maintainers
|
||||
|
||||
# OpenVINO Func Tests:
|
||||
/src/tests/ @openvinotoolkit/openvino-ie-tests-maintainers @openvinotoolkit/openvino-ie-test-developers
|
||||
/src/frontends/tests/frontend/shared/ @openvinotoolkit/openvino-ie-tests-maintainers
|
||||
/thirdparty/gtest/ @openvinotoolkit/openvino-ie-tests-maintainers
|
||||
|
||||
# OpenVINO CPU:
|
||||
/src/plugins/intel_cpu/ @openvinotoolkit/openvino-ie-cpu-maintainers @openvinotoolkit/openvino-ie-cpu-developers
|
||||
/src/common/snippets/ @openvinotoolkit/openvino-ie-cpu-maintainers
|
||||
/thirdparty/xbyak/ @openvinotoolkit/openvino-ie-cpu-maintainers
|
||||
|
||||
# OpenVINO LPT
|
||||
/src/common/low_precision_transformations/ @openvinotoolkit/openvino-ie-lpt-maintainers
|
||||
|
||||
# OpenVINO GPU:
|
||||
/src/plugins/intel_gpu/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
|
||||
/src/tests/**/gpu/ @openvinotoolkit/openvino-ie-gpu-maintainers
|
||||
/thirdparty/ocl/ @openvinotoolkit/openvino-ie-gpu-maintainers @openvinotoolkit/openvino-ie-gpu-developers
|
||||
|
||||
# OpenVINO NPU:
|
||||
/src/plugins/intel_npu/ @openvinotoolkit/openvino-npu-maintainers @openvinotoolkit/openvino-npu-developers
|
||||
|
||||
# OpenVINO Auto (MULTI) plugin:
|
||||
/src/plugins/auto/ @openvinotoolkit/openvino-ie-auto-multi-maintainers
|
||||
|
||||
# OpenVINO Auto (Batch) plugin:
|
||||
/src/plugins/auto_batch/ @openvinotoolkit/openvino-auto-batch-maintainers
|
||||
|
||||
# OpenVINO Hetero plugin:
|
||||
/src/plugins/hetero/ @openvinotoolkit/openvino-hetero-maintainers
|
||||
|
||||
# OpenVINO Template plugin:
|
||||
/src/plugins/template/ @openvinotoolkit/openvino-ie-template-maintainers
|
||||
|
||||
# OpenVINO Frontends:
|
||||
/src/frontends/common/ @openvinotoolkit/openvino-common-frontend-maintainers
|
||||
/src/frontends/ir/ @openvinotoolkit/openvino-ir-frontend-maintainers
|
||||
/src/frontends/paddle/ @openvinotoolkit/openvino-ie-paddle-maintainers
|
||||
/src/frontends/tensorflow/ @openvinotoolkit/openvino-tf-frontend-maintainers
|
||||
/src/frontends/tensorflow_common/ @openvinotoolkit/openvino-tf-frontend-maintainers
|
||||
/src/frontends/tensorflow_lite/ @openvinotoolkit/openvino-tf-frontend-maintainers
|
||||
/src/frontends/pytorch/ @openvinotoolkit/openvino-pytorch-frontend-maintainers
|
||||
|
||||
# OpenVINO ONNX Frontend:
|
||||
/src/frontends/onnx/ @openvinotoolkit/openvino-onnx-frontend-maintainers
|
||||
/thirdparty/onnx/ @openvinotoolkit/openvino-onnx-frontend-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
/thirdparty/protobuf/ @openvinotoolkit/openvino-onnx-frontend-maintainers @openvinotoolkit/openvino-tf-frontend-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
|
||||
# QA Tests:
|
||||
/tests/ @openvinotoolkit/openvino-tests-maintainers
|
||||
/tests/layer_tests/ @openvinotoolkit/openvino-tests-maintainers @openvinotoolkit/openvino-mo-maintainers
|
||||
/tests/layer_tests/pytorch_tests/ @openvinotoolkit/openvino-pytorch-frontend-maintainers
|
||||
/tests/layer_tests/tensorflow_tests @openvinotoolkit/openvino-tf-frontend-maintainers
|
||||
/tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers
|
||||
/tests/model_hub_tests @openvinotoolkit/openvino-tf-frontend-maintainers
|
||||
/tests/model_hub_tests/pytorch @openvinotoolkit/openvino-pytorch-frontend-maintainers
|
||||
|
||||
# Tools:
|
||||
/tools/ @openvinotoolkit/openvino-tools-maintainers
|
||||
/tools/benchmark_tool/ @openvinotoolkit/openvino-ie-python-api-maintainers
|
||||
/tools/legacy/ @openvinotoolkit/openvino-samples-maintainers
|
||||
/tools/openvino_dev/ @openvinotoolkit/openvino-tools-maintainers @openvinotoolkit/openvino-ie-python-api-maintainers
|
||||
/tools/mo/ @openvinotoolkit/openvino-mo-maintainers
|
||||
/tools/ovc/ @openvinotoolkit/openvino-mo-maintainers
|
||||
/thirdparty/open_model_zoo/ @openvinotoolkit/omz-maintainers
|
||||
|
||||
# Documentation
|
||||
/docs/ @openvinotoolkit/openvino-docs-maintainers
|
||||
/docs/CMakeLists.txt @openvinotoolkit/openvino-ie-maintainers
|
||||
/**/*.md @openvinotoolkit/openvino-docs-maintainers
|
||||
/**/*.svg @openvinotoolkit/openvino-docs-maintainers
|
||||
/docs/MO_DG/ @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-mo-maintainers
|
||||
/docs/OV_Runtime_UG/ @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
/docs/IE_PLUGIN_DG/ @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
/docs/Extensibility_UG/ @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
/docs/snippets/ @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-ie-maintainers
|
||||
/docs/OV_Runtime_UG/supported_plugins/ARM_CPU.md @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino_contrib-arm_plugin-maintainers
|
||||
/docs/OV_Runtime_UG/supported_plugins/CPU.md @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-ie-cpu-maintainers
|
||||
/docs/OV_Runtime_UG/supported_plugins/GPU*.md @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-ie-gpu-maintainers
|
||||
|
||||
# Configuration management
|
||||
/**/setup.py @openvinotoolkit/openvino-configuration-mgmt
|
||||
/**/requirements*.* @openvinotoolkit/openvino-configuration-mgmt
|
||||
/docs/requirements.txt @openvinotoolkit/openvino-docs-maintainers @openvinotoolkit/openvino-configuration-mgmt
|
||||
|
||||
# CMake scripts
|
||||
/**/cmake/ @openvinotoolkit/openvino-ie-maintainers
|
||||
/**/*.cmake @openvinotoolkit/openvino-ie-maintainers
|
||||
/CMakeLists.txt @openvinotoolkit/openvino-ie-maintainers
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
---
|
||||
name: Bug
|
||||
about: Create a report to help us improve
|
||||
title: "[Bug]"
|
||||
labels: bug, support_request
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
##### System information (version)
|
||||
<!-- Example
|
||||
- OpenVINO => 2020.4
|
||||
- Operating System / Platform => Windows 64 Bit
|
||||
- Compiler => Visual Studio 2017
|
||||
- Problem classification: Model Conversion
|
||||
- Framework: TensorFlow (if applicable)
|
||||
- Model name: ResNet50 (if applicable)
|
||||
-->
|
||||
|
||||
- OpenVINO=> :grey_question:
|
||||
- Operating System / Platform => :grey_question:
|
||||
- Compiler => :grey_question:
|
||||
- Problem classification => :grey_question:
|
||||
|
||||
##### Detailed description
|
||||
<!-- your description -->
|
||||
|
||||
##### Steps to reproduce
|
||||
|
||||
<!--
|
||||
Describe your problem and steps you've done before you got to this point.
|
||||
to add code example fence it with triple backticks and optional file extension
|
||||
```.cpp
|
||||
// C++ code example
|
||||
```
|
||||
or attach as .txt or .zip file
|
||||
-->
|
||||
|
||||
##### Issue submission checklist
|
||||
|
||||
- [ ] I report the issue, it's not a question
|
||||
<!--
|
||||
OpenVINO team works with support forum, Stack Overflow and other communities
|
||||
to discuss problems. Tickets with question without real issue statement will be
|
||||
closed.
|
||||
-->
|
||||
- [ ] I checked the problem with documentation, FAQ, open issues, Stack Overflow, etc and have not found solution
|
||||
<!--
|
||||
Places to check:
|
||||
* OpenVINO documentation: https://docs.openvinotoolkit.org/
|
||||
* OpenVINO forum: https://community.intel.com/t5/Intel-Distribution-of-OpenVINO/bd-p/distribution-openvino-toolkit
|
||||
* OpenVINO issue tracker: https://github.com/openvinotoolkit/openvino/issues?q=is%3Aissue
|
||||
* Stack Overflow branch: https://stackoverflow.com/questions/tagged/openvino
|
||||
-->
|
||||
- [ ] There is reproducer code and related data files: images, videos, models, etc.
|
||||
<!--
|
||||
The best reproducer -- test case for OpenVINO that we can add to the library.
|
||||
-->
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
name: Bug report
|
||||
description: Help us improve OpenVINO.
|
||||
title: "[Bug]: "
|
||||
labels: ["bug", "support_request"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Please provide all the necessary information to expedite the response.
|
||||
- type: input
|
||||
id: ov_version
|
||||
attributes:
|
||||
label: OpenVINO Version
|
||||
description: OpenVINO version, branch, or tag in OpenVINO GitHub
|
||||
placeholder: 2021.4.0 LTS / Master Branch / tag 2022.3.0
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System
|
||||
description: What OS are you using?
|
||||
options:
|
||||
- Ubuntu 18.04 (LTS)
|
||||
- Ubuntu 20.04 (LTS)
|
||||
- Windows System
|
||||
- Red Hat Enterprise Linux 8
|
||||
- Android System
|
||||
- Raspbian Stretch OS
|
||||
- macOS Systems for Intel CPU
|
||||
- macOS Systems for Apple Silicon
|
||||
- WebAssembly
|
||||
- Other (Please specify in description)
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: device_use
|
||||
attributes:
|
||||
label: Device used for inference
|
||||
description: What hardware are you using for inference?
|
||||
options:
|
||||
- CPU
|
||||
- GPU
|
||||
- NPU
|
||||
- AUTO
|
||||
- HETERO
|
||||
- BATCH
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: framework
|
||||
attributes:
|
||||
label: Framework
|
||||
description: Framework used for model optimization
|
||||
options:
|
||||
- TensorFlow 1
|
||||
- Keras (TensorFlow 2)
|
||||
- Caffe
|
||||
- ONNX
|
||||
- PyTorch
|
||||
- mxnet
|
||||
- PaddlePaddle
|
||||
validations:
|
||||
required: false
|
||||
- type: input
|
||||
id: model_name
|
||||
attributes:
|
||||
label: Model used
|
||||
description: Link to the model
|
||||
placeholder: ResNet50 / YOLOv4
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: bug_description
|
||||
attributes:
|
||||
label: Issue description
|
||||
description: What issue are you having, and what did you expect to happen instead?
|
||||
placeholder: "Error when performing model optimization on yolov4 model."
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: step_by_step
|
||||
attributes:
|
||||
label: Step-by-step reproduction
|
||||
description: How can we reproduce your issue?
|
||||
placeholder: Please provide detailed instructions on how to reproduce the issue
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Relevant log output
|
||||
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
|
||||
render: shell
|
||||
- type: checkboxes
|
||||
id: terms
|
||||
attributes:
|
||||
label: Issue submission checklist
|
||||
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/intel/intel-one-mono/blob/main/CODE_OF_CONDUCT.md)
|
||||
options:
|
||||
- label: I'm reporting an issue. It's not a question.
|
||||
required: true
|
||||
- label: I checked the problem with the documentation, FAQ, open issues, Stack Overflow, etc., and have not found a solution.
|
||||
required: true
|
||||
- label: There is reproducer code and related data files such as images, videos, models, etc.
|
||||
required: true
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
name: Build Issue report
|
||||
description: Report a build or installation issue.
|
||||
title: "[Build]: "
|
||||
labels: ["build", "support_request"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Please provide all the necessary information to expedite the response.
|
||||
- type: input
|
||||
id: ov_version
|
||||
attributes:
|
||||
label: OpenVINO Version
|
||||
description: OpenVINO version, branch, or tag in OpenVINO GitHub
|
||||
placeholder: 2021.4.0 LTS / Master Branch / tag 2022.3.0
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System
|
||||
description: What OS are you using?
|
||||
options:
|
||||
- Ubuntu 18.04 (LTS)
|
||||
- Ubuntu 20.04 (LTS)
|
||||
- Ubuntu 22.04 (LTS)
|
||||
- Windows System
|
||||
- Red Hat Enterprise Linux 8
|
||||
- OpenSUSE
|
||||
- Android System
|
||||
- Raspbian Stretch OS
|
||||
- macOS Systems for Intel CPU
|
||||
- macOS Systems for Apple Silicon
|
||||
- WebAssembly
|
||||
- WSL2 for Windows
|
||||
- Other (Please specify in description)
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: architecture
|
||||
attributes:
|
||||
label: Hardware Architecture
|
||||
description: What is your hardware architecture used in this test?
|
||||
options:
|
||||
- x86 (64 bits)
|
||||
- x86 (32 bits)
|
||||
- ARM (64 bits)
|
||||
- ARM (32 bits)
|
||||
- RISC-V
|
||||
- Other (please specify in the description)
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: target_platform
|
||||
attributes:
|
||||
label: Target Platform
|
||||
description: |
|
||||
You can also provide us full system log with the following command
|
||||
Windows cmd - "systeminfo"
|
||||
Linux terminal - "lscpu" and "lscpu -e"
|
||||
placeholder: Paste your full platform/system information here
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: build_description
|
||||
attributes:
|
||||
label: Build issue description
|
||||
description: What issue are you facing during the build/installation?
|
||||
placeholder: Please provide a detailed description of what happened
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: build_script
|
||||
attributes:
|
||||
label: Build script or step-by-step to reproduce
|
||||
description: How can we reproduce your issue?
|
||||
placeholder: Please provide detailed instructions on how to reproduce the issue
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: build_logs
|
||||
attributes:
|
||||
label: Relevant log output
|
||||
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so there is no need for backticks.
|
||||
render: shell
|
||||
- type: checkboxes
|
||||
id: terms
|
||||
attributes:
|
||||
label: Issue submission checklist
|
||||
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/intel/intel-one-mono/blob/main/CODE_OF_CONDUCT.md)
|
||||
options:
|
||||
- label: I'm reporting an issue. It's not a question.
|
||||
required: true
|
||||
- label: I checked the problem with the documentation, FAQ, open issues, Stack Overflow, etc., and have not found a solution.
|
||||
required: true
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
name: Documentation issue report
|
||||
description: Report an issue with Documentation.
|
||||
title: "[Docs]: "
|
||||
labels: ["docs", "support_request"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Please provide all the necessary information to expedite the response.
|
||||
- type: input
|
||||
id: doc_link
|
||||
attributes:
|
||||
label: Documentation link
|
||||
description: Please provide the link for the documentation issue
|
||||
placeholder: e.g. intel.com/content/www/us/en/developer/tools/openvino-toolkit/system-requirements.html
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Description
|
||||
description: Provide a description of the issue you noticed.
|
||||
validations:
|
||||
required: true
|
||||
- type: checkboxes
|
||||
id: terms
|
||||
attributes:
|
||||
label: Issue submission checklist
|
||||
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/intel/intel-one-mono/blob/main/CODE_OF_CONDUCT.md)
|
||||
options:
|
||||
- label: I'm reporting a documentation issue. It's not a question.
|
||||
required: true
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
name: Feature request
|
||||
description: Suggest a feature or improvement for the OpenVINO toolkit.
|
||||
title: "[Feature Request]: "
|
||||
labels: ["enhancement", "feature"]
|
||||
assignees:
|
||||
- octocat
|
||||
body:
|
||||
- type: textarea
|
||||
id: request_description
|
||||
attributes:
|
||||
label: Request Description
|
||||
description: What would you like us to improve on?
|
||||
placeholder: Please provide a detailed description of your request.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: feature_usecase
|
||||
attributes:
|
||||
label: Feature Use Case
|
||||
description: What is the use case of the feature you are proposing?
|
||||
placeholder: What is the new feature use case? How will it be useful?
|
||||
validations:
|
||||
required: false
|
||||
- type: checkboxes
|
||||
id: check2
|
||||
attributes:
|
||||
label: Issue submission checklist
|
||||
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/intel/intel-one-mono/blob/main/CODE_OF_CONDUCT.md)
|
||||
options:
|
||||
- label: The feature request or improvement must be related to OpenVINO
|
||||
required: true
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
name: Good First Issue
|
||||
description: Create a Good First Issue for new contributors.
|
||||
title: "[Good First Issue]: "
|
||||
labels: ["good first issue", "no_stale"]
|
||||
body:
|
||||
- type: textarea
|
||||
id: context
|
||||
attributes:
|
||||
label: Context
|
||||
description: |
|
||||
Let the contributors know what your component is responsible for,
|
||||
what's the importance of the change and why it's needed.
|
||||
Keep in mind the Good First Issue is for new contributors.
|
||||
placeholder: What is it and why is it important?
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: todo_list
|
||||
attributes:
|
||||
label: What needs to be done?
|
||||
description: |
|
||||
Be as verbose as possible, provide a TODO list if viable.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: example_prs
|
||||
attributes:
|
||||
label: Example Pull Requests
|
||||
description: |
|
||||
Provide example Pull requests, if there are any.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: resources
|
||||
attributes:
|
||||
label: Resources
|
||||
description: |
|
||||
Any materials related to the task, such as operator specifications,
|
||||
discussions, guides.
|
||||
value: |
|
||||
- [Contribution guide - start here!](https://github.com/openvinotoolkit/openvino/blob/master/CONTRIBUTING.md)
|
||||
- [Intel DevHub Discord channel](https://discord.gg/7pVRxUwdWG) - engage in discussions, ask questions and talk to OpenVINO developers
|
||||
- [How to link your Pull Request to an issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue-using-the-pull-request-sidebar)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: contact_points
|
||||
attributes:
|
||||
label: Contact points
|
||||
description: |
|
||||
People who can be asked questions about the task.
|
||||
placeholder: GitHub users
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: ticket
|
||||
attributes:
|
||||
label: Ticket
|
||||
description: |
|
||||
Provide the ticket number, if available.
|
||||
validations:
|
||||
required: false
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
name: Performance Issue Report
|
||||
description: This report is for the performance-related issue
|
||||
title: "[Performance]: "
|
||||
labels: ["performance", "support_request"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Please provide all the necessary information to expedite the response.
|
||||
- type: input
|
||||
id: ov_version
|
||||
attributes:
|
||||
label: OpenVINO Version
|
||||
description: OpenVINO version, branch, or tag in OpenVINO GitHub
|
||||
placeholder: 2021.4.0 LTS / Master Branch / tag 2022.3.0
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System
|
||||
description: What OS are you using?
|
||||
options:
|
||||
- Ubuntu 18.04 (LTS)
|
||||
- Ubuntu 20.04 (LTS)
|
||||
- Ubuntu 22.04 (LTS)
|
||||
- Windows System
|
||||
- Red Hat Enterprise Linux 8
|
||||
- OpenSUSE
|
||||
- Android System
|
||||
- Raspbian Stretch OS
|
||||
- macOS Systems for Intel CPU
|
||||
- macOS Systems for Apple Silicon
|
||||
- WebAssembly
|
||||
- WSL2 on Windows
|
||||
- Other (Please specify in description)
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: device_use
|
||||
attributes:
|
||||
label: Device used for inference
|
||||
description: What hardware are you using for inference?
|
||||
options:
|
||||
- CPU
|
||||
- iGPU
|
||||
- dGPU
|
||||
- NPU
|
||||
validations:
|
||||
required: false
|
||||
- type: dropdown
|
||||
id: openvino_installation
|
||||
attributes:
|
||||
label: OpenVINO installation
|
||||
description: How do you install OpenVINO on your system?
|
||||
options:
|
||||
- PyPi
|
||||
- Docker
|
||||
- Build from source
|
||||
- Other virtual machines
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: openvino_api
|
||||
attributes:
|
||||
label: Programming Language
|
||||
description: What is the programming language you use in your performance test?
|
||||
options:
|
||||
- Python
|
||||
- C++
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: architecture
|
||||
attributes:
|
||||
label: Hardware Architecture
|
||||
description: What is your hardware architecture used in this test?
|
||||
options:
|
||||
- x86 (64 bits)
|
||||
- x86 (32 bits)
|
||||
- ARM (64 bits)
|
||||
- ARM (32 bits)
|
||||
- RISC-V
|
||||
- Other (please specify in the description)
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: model_name
|
||||
attributes:
|
||||
label: Model used
|
||||
description: Link to the model
|
||||
placeholder: ResNet50 / YOLOv4
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: model_quantized
|
||||
attributes:
|
||||
label: Model quantization
|
||||
description: Is your model quantized?
|
||||
options:
|
||||
- 'Yes'
|
||||
- 'No'
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: target_platform
|
||||
attributes:
|
||||
label: Target Platform
|
||||
description: |
|
||||
You can also provide us full system log with the following command
|
||||
Windows cmd - "systeminfo"
|
||||
Linux terminal - "lscpu" and "lscpu -e"
|
||||
placeholder: Paste your full platform/system information here
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: performance_description
|
||||
attributes:
|
||||
label: Performance issue description
|
||||
description: What issue are you having, and what did you expect to happen instead?
|
||||
placeholder: |
|
||||
Please provide a detailed description of what happened.
|
||||
Can the issue be reproduced using benchmark_app?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: step_by_step
|
||||
attributes:
|
||||
label: Step-by-step reproduction
|
||||
description: How can we reproduce your issue?
|
||||
placeholder: Please provide detailed instructions on how to reproduce the issue
|
||||
validations:
|
||||
required: false
|
||||
- type: checkboxes
|
||||
id: terms
|
||||
attributes:
|
||||
label: Issue submission checklist
|
||||
description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/intel/intel-one-mono/blob/main/CODE_OF_CONDUCT.md)
|
||||
options:
|
||||
- label: I'm reporting a performance issue. It's not a question.
|
||||
required: true
|
||||
- label: I checked the problem with the documentation, FAQ, open issues, Stack Overflow, etc., and have not found a solution.
|
||||
required: true
|
||||
- label: There is reproducer code and related data files such as images, videos, models, etc.
|
||||
required: true
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
name: Pre-release Feedback
|
||||
description: Feedback on Pre-release OpenVINO
|
||||
title: "[Pre-Release Feedback]:"
|
||||
labels: ["Pre-release", "support_request"]
|
||||
body:
|
||||
- type: input
|
||||
id: pre_version
|
||||
attributes:
|
||||
label: Pre-release Version
|
||||
description: What is the pre-release version you are using?
|
||||
placeholder: 2023.0.0.dev20230427
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: feedback
|
||||
attributes:
|
||||
label: Pre-release feedback
|
||||
description: What is the issue or feedback on the pre-release?
|
||||
placeholder: There is an inference performance drop in OpenVINO 2022.4.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: thoughts
|
||||
attributes:
|
||||
label: New Feature Feedback
|
||||
description: Do you have any feedback on the new features released in the pre-release?
|
||||
placeholder: Any thoughts on the new features are welcome.
|
||||
validations:
|
||||
required: false
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/intel/intel-one-mono/blob/main/CODE_OF_CONDUCT.md)
|
||||
Have a new feature you would like to see in OpenVINO? File a feature request <a href="https://github.com/openvinotoolkit/openvino/issues/new/choose">HERE</a>.
|
||||
You can also implement the feature by following the <a href="https://github.com/openvinotoolkit/openvino/blob/master/CONTRIBUTING.md">CONTRIBUTING.MD</a>
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
lib/
|
||||
dist/
|
||||
node_modules/
|
||||
coverage/
|
||||
|
|
@ -1 +0,0 @@
|
|||
20.6.0
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
dist/
|
||||
node_modules/
|
||||
coverage/
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
{
|
||||
"printWidth": 80,
|
||||
"tabWidth": 2,
|
||||
"useTabs": false,
|
||||
"semi": true,
|
||||
"singleQuote": true,
|
||||
"quoteProps": "as-needed",
|
||||
"jsxSingleQuote": false,
|
||||
"trailingComma": "none",
|
||||
"bracketSpacing": true,
|
||||
"bracketSameLine": true,
|
||||
"arrowParens": "avoid",
|
||||
"proseWrap": "always",
|
||||
"htmlWhitespaceSensitivity": "css",
|
||||
"endOfLine": "lf"
|
||||
}
|
||||
|
|
@ -1,286 +0,0 @@
|
|||
/**
|
||||
* Unit tests for the action's main functionality, src/cleanupImpl.js
|
||||
*/
|
||||
const core = require('@actions/core');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const fs = require('fs');
|
||||
const cleanupImpl = require('../src/cleanupImpl');
|
||||
|
||||
// Mock the GitHub Actions core library
|
||||
const getInputMock = jest.spyOn(core, 'getInput').mockImplementation();
|
||||
const setFailedMock = jest.spyOn(core, 'setFailed').mockImplementation();
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-cleanup-'));
|
||||
const cacheRemotePath = path.join(tempDir, 'cache_remote');
|
||||
|
||||
const cacheFiles = ['cache_1.cache', 'cache_2.cache', 'cache_3.cache'];
|
||||
const minAccessTime = 7 * 24 * 60 * 60 * 1000; // 1 week
|
||||
|
||||
// Mock the action's main function
|
||||
const runMock = jest.spyOn(cleanupImpl, 'cleanUp');
|
||||
|
||||
describe('cleanup', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Set up file system before each test
|
||||
fs.mkdirSync(cacheRemotePath, { recursive: true });
|
||||
|
||||
const fileSizeInBytes = 1024 * 1024 * 1024; // 1 GB
|
||||
const buffer = Buffer.alloc(fileSizeInBytes);
|
||||
let id = 1;
|
||||
for (const cache of cacheFiles) {
|
||||
const cachePath = path.join(cacheRemotePath, cache);
|
||||
fs.writeFileSync(cachePath, buffer);
|
||||
fs.utimesSync(
|
||||
cachePath,
|
||||
new Date(Date.now() - minAccessTime * id++),
|
||||
new Date()
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up mock file system after each test
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
it('Cleanup old files using restore key', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 1;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
// cache2 and cache3 should be removed
|
||||
for (const cache of cacheFiles.slice(1, 2)) {
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cache))).toBe(false);
|
||||
}
|
||||
// check that file1 exists
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cacheFiles[0]))).toBe(true);
|
||||
});
|
||||
|
||||
it('Cleanup old files using key', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 1;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
// cache2 and cache3 should be removed
|
||||
for (const cache of cacheFiles.slice(1, 2)) {
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cache))).toBe(false);
|
||||
}
|
||||
// check that file1 exists
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cacheFiles[0]))).toBe(true);
|
||||
});
|
||||
|
||||
it('Skip cleanup old files by size', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 5;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
for (const cache of cacheFiles) {
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cache))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('Skip cleanup old files by atime', async () => {
|
||||
for (const cache of cacheFiles) {
|
||||
const cachePath = path.join(cacheRemotePath, cache);
|
||||
fs.utimesSync(
|
||||
cachePath,
|
||||
new Date(Date.now() - minAccessTime / 2),
|
||||
new Date()
|
||||
);
|
||||
}
|
||||
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 1;
|
||||
case 'max-cache-size':
|
||||
return 5;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
for (const cache of cacheFiles) {
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cache))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('Cleanup recently used files by max cache limit', async () => {
|
||||
for (const cache of cacheFiles) {
|
||||
const cachePath = path.join(cacheRemotePath, cache);
|
||||
fs.utimesSync(
|
||||
cachePath,
|
||||
new Date(Date.now() - minAccessTime / 2),
|
||||
new Date()
|
||||
);
|
||||
}
|
||||
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 1;
|
||||
case 'max-cache-size':
|
||||
return 2;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
// cache2 and cache3 should be removed
|
||||
for (const cache of cacheFiles.slice(1, 2)) {
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cache))).toBe(false);
|
||||
}
|
||||
// check that file1 exists
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cacheFiles[0]))).toBe(true);
|
||||
});
|
||||
|
||||
it('Test unexpected behaviour', async () => {
|
||||
// Set folder permissions to read-only
|
||||
fs.chmodSync(cacheRemotePath, '555');
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 2;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setFailedMock).not.toHaveBeenCalled();
|
||||
|
||||
fs.chmodSync(cacheRemotePath, '777');
|
||||
});
|
||||
|
||||
it('Cleanup absent directory', async () => {
|
||||
const cachePathAbsent = path.join(tempDir, 'cache_remote_absent');
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cachePathAbsent;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 2;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setFailedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('Cleanup directory with subdirectory', async () => {
|
||||
const cacheSubPath = path.join(cacheRemotePath, 'cache_subdir.cache');
|
||||
fs.mkdirSync(cacheSubPath, { recursive: true });
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'cache-size':
|
||||
return 1;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await cleanupImpl.cleanUp();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
// cache2 and cache3 should be removed
|
||||
for (const cache of cacheFiles.slice(1, 2)) {
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cache))).toBe(false);
|
||||
}
|
||||
// check that file1 exists
|
||||
expect(fs.existsSync(path.join(cacheRemotePath, cacheFiles[0]))).toBe(true);
|
||||
// check that sub directory exists
|
||||
expect(fs.existsSync(cacheSubPath)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,266 +0,0 @@
|
|||
/**
|
||||
* Unit tests for the action's main functionality, src/restoreImpl.js
|
||||
*/
|
||||
const core = require('@actions/core');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const fs = require('fs');
|
||||
const tar = require('tar');
|
||||
const restoreImpl = require('../src/restoreImpl');
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-restore-'));
|
||||
const cacheLocalPath = path.join(tempDir, 'cache_local');
|
||||
const cacheRemotePath = path.join(tempDir, 'cache_remote');
|
||||
const cacheTmpPath = path.join(tempDir, 'cache_tmp');
|
||||
|
||||
const cacheFiles = ['cache_1.cache', 'cache_2.cache', 'cache_3.cache'];
|
||||
const testFiles = ['file1.txt', 'file2.txt', 'file3.txt'];
|
||||
|
||||
// Mock the GitHub Actions core library
|
||||
const getInputMock = jest.spyOn(core, 'getInput').mockImplementation();
|
||||
const setOutputMock = jest.spyOn(core, 'setOutput').mockImplementation();
|
||||
const setFailedMock = jest.spyOn(core, 'setFailed').mockImplementation();
|
||||
|
||||
// Clean up mock file system after each test
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
// Mock the action's main function
|
||||
const runMock = jest.spyOn(restoreImpl, 'restore');
|
||||
|
||||
describe('restore', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Set up mock file system before each test
|
||||
fs.mkdirSync(cacheLocalPath, { recursive: true });
|
||||
fs.mkdirSync(cacheRemotePath, { recursive: true });
|
||||
fs.mkdirSync(cacheTmpPath, { recursive: true });
|
||||
|
||||
// Define test files
|
||||
const file1Path = path.join(cacheTmpPath, testFiles[0]);
|
||||
fs.writeFileSync(file1Path, 'File 1 contents');
|
||||
tar.c(
|
||||
{
|
||||
gzip: true,
|
||||
file: cacheFiles[0],
|
||||
cwd: cacheTmpPath,
|
||||
sync: true
|
||||
},
|
||||
['.']
|
||||
);
|
||||
fs.renameSync(cacheFiles[0], path.join(cacheRemotePath, cacheFiles[0]));
|
||||
fs.utimesSync(
|
||||
path.join(cacheRemotePath, cacheFiles[0]),
|
||||
new Date(Date.now() - 1500),
|
||||
new Date(Date.now() - 1500)
|
||||
);
|
||||
|
||||
//
|
||||
const file2Path = path.join(cacheTmpPath, testFiles[1]);
|
||||
fs.writeFileSync(file2Path, 'File 2 contents');
|
||||
tar.c(
|
||||
{
|
||||
gzip: true,
|
||||
file: cacheFiles[1],
|
||||
cwd: cacheTmpPath,
|
||||
sync: true
|
||||
},
|
||||
['.']
|
||||
);
|
||||
fs.renameSync(cacheFiles[1], path.join(cacheRemotePath, cacheFiles[1]));
|
||||
fs.utimesSync(
|
||||
path.join(cacheRemotePath, cacheFiles[1]),
|
||||
new Date(Date.now() - 1000),
|
||||
new Date(Date.now() - 1000)
|
||||
);
|
||||
//
|
||||
const file3Path = path.join(cacheTmpPath, testFiles[2]);
|
||||
fs.writeFileSync(file3Path, 'File 3 contents');
|
||||
tar.c(
|
||||
{
|
||||
gzip: true,
|
||||
file: cacheFiles[2],
|
||||
cwd: cacheTmpPath,
|
||||
sync: true
|
||||
},
|
||||
['.']
|
||||
);
|
||||
fs.renameSync(cacheFiles[2], path.join(cacheRemotePath, cacheFiles[2]));
|
||||
fs.utimesSync(
|
||||
path.join(cacheRemotePath, cacheFiles[2]),
|
||||
new Date(Date.now() - 500),
|
||||
new Date(Date.now() - 500)
|
||||
);
|
||||
});
|
||||
|
||||
it('gets the cache file', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await restoreImpl.restore();
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
// Verify that all of the core library functions were called correctly
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'cache-file',
|
||||
cacheFiles[2]
|
||||
);
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(2, 'cache-hit', true);
|
||||
let id = 1;
|
||||
for (const filename of testFiles) {
|
||||
const filePath = path.join(cacheLocalPath, filename);
|
||||
expect(fs.existsSync(filePath)).toBe(true);
|
||||
const fileContent = fs.readFileSync(filePath, 'utf8');
|
||||
expect(fileContent).toBe(`File ${id++} contents`);
|
||||
}
|
||||
});
|
||||
|
||||
it('gets the updated cache file', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
// updated cache 2 access time
|
||||
fs.utimesSync(
|
||||
path.join(cacheRemotePath, cacheFiles[1]),
|
||||
new Date(Date.now() - 100),
|
||||
new Date(Date.now() - 100)
|
||||
);
|
||||
|
||||
await restoreImpl.restore();
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
// Verify that all of the core library functions were called correctly
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'cache-file',
|
||||
cacheFiles[1]
|
||||
);
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(2, 'cache-hit', true);
|
||||
let id = 1;
|
||||
for (const filename of testFiles.slice(0, 1)) {
|
||||
const filePath = path.join(cacheLocalPath, filename);
|
||||
expect(fs.existsSync(filePath)).toBe(true);
|
||||
const fileContent = fs.readFileSync(filePath, 'utf8');
|
||||
expect(fileContent).toBe(`File ${id++} contents`);
|
||||
}
|
||||
// check that file3 is absent
|
||||
expect(fs.existsSync(path.join(cacheLocalPath, testFiles[2]))).toBe(false);
|
||||
});
|
||||
|
||||
it('gets the cache file and extract to absent dir', async () => {
|
||||
const cacheAbsentPath = path.join(tempDir, 'cache_absent');
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'restore-keys':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheAbsentPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await restoreImpl.restore();
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
// Verify that all of the core library functions were called correctly
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'cache-file',
|
||||
cacheFiles[2]
|
||||
);
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(2, 'cache-hit', true);
|
||||
let id = 1;
|
||||
for (const filename of testFiles) {
|
||||
const filePath = path.join(cacheAbsentPath, filename);
|
||||
expect(fs.existsSync(filePath)).toBe(true);
|
||||
const fileContent = fs.readFileSync(filePath, 'utf8');
|
||||
expect(fileContent).toBe(`File ${id++} contents`);
|
||||
}
|
||||
});
|
||||
|
||||
it('Test when no cache found', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'key';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await restoreImpl.restore();
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
// Verify that all of the core library functions were called correctly
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(1, 'cache-file', '');
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(2, 'cache-hit', false);
|
||||
for (const filename of testFiles) {
|
||||
const filePath = path.join(cacheLocalPath, filename);
|
||||
expect(fs.existsSync(filePath)).toBe(false);
|
||||
}
|
||||
expect(setFailedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('Unsupported cache found', async () => {
|
||||
const cacheFakePath = path.join(cacheRemotePath, 'cache-fake.cache');
|
||||
fs.writeFileSync(cacheFakePath, 'Fake content');
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await restoreImpl.restore();
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setOutputMock).not.toHaveBeenCalled();
|
||||
expect(setFailedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,190 +0,0 @@
|
|||
/**
|
||||
* Unit tests for the action's main functionality, src/saveImpl.js
|
||||
*/
|
||||
const core = require('@actions/core');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const fs = require('fs');
|
||||
const saveImpl = require('../src/saveImpl');
|
||||
|
||||
// Mock the GitHub Actions core library
|
||||
const getInputMock = jest.spyOn(core, 'getInput').mockImplementation();
|
||||
const setOutputMock = jest.spyOn(core, 'setOutput').mockImplementation();
|
||||
const setFailedMock = jest.spyOn(core, 'setFailed').mockImplementation();
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-'));
|
||||
const cacheLocalPath = path.join(tempDir, 'cache_local');
|
||||
const cacheRemotePath = path.join(tempDir, 'cache_remote');
|
||||
|
||||
// Clean up mock file system after each test
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
// Mock the action's main function
|
||||
const runMock = jest.spyOn(saveImpl, 'save');
|
||||
|
||||
describe('save', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Set up mock file system before each test
|
||||
|
||||
// Create cache pathes
|
||||
fs.mkdirSync(cacheLocalPath, { recursive: true });
|
||||
fs.mkdirSync(cacheRemotePath, { recursive: true });
|
||||
|
||||
// Define test files
|
||||
const file1Path = path.join(cacheLocalPath, 'file1.txt');
|
||||
const file2Path = path.join(cacheLocalPath, 'file2.txt');
|
||||
|
||||
// Write content to the test files
|
||||
fs.writeFileSync(file1Path, 'File 1 contents');
|
||||
fs.writeFileSync(file2Path, 'File 2 contents');
|
||||
});
|
||||
|
||||
it('Cache files pass', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await saveImpl.save();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'cache-file',
|
||||
'cache.cache'
|
||||
);
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(2, 'cache-hit', true);
|
||||
|
||||
expect(fs.existsSync(`${cacheRemotePath}/cache.cache`)).toBe(true); // Check if the tar file was created
|
||||
});
|
||||
|
||||
it('Cache files: absent local cache dir', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
const cachePathAbsent = path.join(tempDir, 'cache_local_absent');
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cachePathAbsent;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await saveImpl.save();
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(fs.existsSync(`${cacheRemotePath}/cache.cache`)).toBe(false); // Check that the tar file was not created
|
||||
});
|
||||
|
||||
it('Cache files: absent remote cache dir', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
const cacheRemotePathAbsent = path.join(tempDir, 'cache_remote_absent');
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePathAbsent;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await saveImpl.save();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'cache-file',
|
||||
'cache.cache'
|
||||
);
|
||||
expect(setOutputMock).toHaveBeenNthCalledWith(2, 'cache-hit', true);
|
||||
expect(fs.existsSync(`${cacheRemotePathAbsent}/cache.cache`)).toBe(true); // Check if the tar file was created
|
||||
});
|
||||
|
||||
it('Cache files: key is not set', async () => {
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await saveImpl.save();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setOutputMock).not.toHaveBeenCalled();
|
||||
expect(setFailedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('Cache files: file already exists', async () => {
|
||||
const cacheFakePath = path.join(cacheRemotePath, 'cache-fake.cache');
|
||||
fs.writeFileSync(cacheFakePath, 'Fake content');
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache-fake';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await saveImpl.save();
|
||||
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setOutputMock).not.toHaveBeenCalled();
|
||||
expect(setFailedMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('Test unexpected behaviour', async () => {
|
||||
// Set folder permissions to read-only
|
||||
fs.chmodSync(cacheRemotePath, '555');
|
||||
// Set the action's inputs as return values from core.getInput()
|
||||
getInputMock.mockImplementation(name => {
|
||||
switch (name) {
|
||||
case 'cache-path':
|
||||
return cacheRemotePath;
|
||||
case 'key':
|
||||
return 'cache';
|
||||
case 'path':
|
||||
return cacheLocalPath;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
await saveImpl.save();
|
||||
expect(runMock).toHaveReturned();
|
||||
expect(setOutputMock).not.toHaveBeenCalled();
|
||||
expect(setFailedMock).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
name: 'Cache'
|
||||
description: 'Cache to the local file system'
|
||||
author: 'Mikhail Ryzhov'
|
||||
|
||||
inputs:
|
||||
cache-path:
|
||||
description: 'A local path where to search and save caches'
|
||||
required: true
|
||||
path:
|
||||
description:
|
||||
'A list of files, directories, and wildcard patterns to cache and restore'
|
||||
required: true
|
||||
key:
|
||||
description: 'An explicit key for restoring and saving the cache'
|
||||
required: true
|
||||
restore-keys:
|
||||
description:
|
||||
'An ordered list of keys to use for restoring stale cache if no cache hit
|
||||
occurred for key.'
|
||||
default: ''
|
||||
required: false
|
||||
save-always:
|
||||
description:
|
||||
'Run the post step to save the cache even if another step before fails'
|
||||
default: false
|
||||
required: false
|
||||
cleanup-always:
|
||||
description:
|
||||
'Run the post cleanup step to remove old cache files even if another step
|
||||
before fails'
|
||||
default: false
|
||||
required: false
|
||||
cache-size:
|
||||
description:
|
||||
'Allowed cache storage size in Gb. Least recently used caches will be
|
||||
automatically evicted to limit the total cache storage. The cache size can
|
||||
exceed this value, but it is limited by "max-cache-size" value'
|
||||
default: 10
|
||||
required: false
|
||||
max-cache-size:
|
||||
description:
|
||||
'Maximum cache storage size in Gb. All cache files will be automatically
|
||||
evicted to limit the total cache storage'
|
||||
default: 50
|
||||
required: false
|
||||
|
||||
outputs:
|
||||
cache-hit:
|
||||
description:
|
||||
'A boolean value to indicate an exact match was found for the primary key'
|
||||
cache-file:
|
||||
description: 'Found cache file'
|
||||
|
||||
runs:
|
||||
using: node20
|
||||
main: 'dist/restore/index.js'
|
||||
post: 'dist/save/index.js'
|
||||
post-if:
|
||||
'success() || github.event.inputs.save-always ||
|
||||
github.event.inputs.cleanup-always'
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
name: 'Cleanup Cache'
|
||||
description: 'Remove old cache artifacts'
|
||||
author: 'OpenVINO Developers'
|
||||
|
||||
inputs:
|
||||
cache-path:
|
||||
description: 'A local path where to search and save caches'
|
||||
required: true
|
||||
key:
|
||||
description: 'An explicit key for restoring and saving the cache'
|
||||
required: true
|
||||
restore-keys:
|
||||
description:
|
||||
'An ordered list of keys to use for restoring stale cache if no cache hit
|
||||
occurred for key.'
|
||||
default: ''
|
||||
required: false
|
||||
cache-size:
|
||||
description:
|
||||
'Allowed cache storage size in Gb. Least recently used caches will be
|
||||
automatically evicted to limit the total cache storage. The cache size can
|
||||
exceed this value, but it is limited by "max-cache-size" value'
|
||||
default: 10
|
||||
required: false
|
||||
max-cache-size:
|
||||
description:
|
||||
'Maximum cache storage size in Gb. All cache files will be automatically
|
||||
evicted to limit the total cache storage'
|
||||
default: 50
|
||||
required: false
|
||||
|
||||
runs:
|
||||
using: 'node20'
|
||||
main: '../dist/cleanup-only/index.js'
|
||||
branding:
|
||||
icon: 'archive'
|
||||
color: 'gray-dark'
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,50 +0,0 @@
|
|||
env:
|
||||
commonjs: true
|
||||
es6: true
|
||||
jest: true
|
||||
node: true
|
||||
|
||||
globals:
|
||||
Atomics: readonly
|
||||
SharedArrayBuffer: readonly
|
||||
|
||||
ignorePatterns:
|
||||
- '!.*'
|
||||
- '**/node_modules/.*'
|
||||
- '**/dist/.*'
|
||||
- '**/coverage/.*'
|
||||
- '*.json'
|
||||
|
||||
parser: '@babel/eslint-parser'
|
||||
|
||||
parserOptions:
|
||||
ecmaVersion: 2023
|
||||
sourceType: module
|
||||
requireConfigFile: false
|
||||
babelOptions:
|
||||
babelrc: false
|
||||
configFile: false
|
||||
presets:
|
||||
- jest
|
||||
|
||||
plugins:
|
||||
- jest
|
||||
|
||||
extends:
|
||||
- eslint:recommended
|
||||
- plugin:github/recommended
|
||||
- plugin:jest/recommended
|
||||
|
||||
rules:
|
||||
{
|
||||
'camelcase': 'warn',
|
||||
'eslint-comments/no-use': 'off',
|
||||
'eslint-comments/no-unused-disable': 'off',
|
||||
'i18n-text/no-en': 'off',
|
||||
'import/no-commonjs': 'off',
|
||||
'import/no-namespace': 'off',
|
||||
'no-console': 'off',
|
||||
'no-unused-vars': 'warn',
|
||||
'prettier/prettier': 'error',
|
||||
'semi': 'error'
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Unordered list style
|
||||
MD004:
|
||||
style: dash
|
||||
|
||||
# Ordered list item prefix
|
||||
MD029:
|
||||
style: one
|
||||
|
||||
# Spaces after list markers
|
||||
MD030:
|
||||
ul_single: 1
|
||||
ol_single: 1
|
||||
ul_multi: 1
|
||||
ol_multi: 1
|
||||
|
||||
# Code block style
|
||||
MD046:
|
||||
style: fenced
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
rules:
|
||||
document-end: disable
|
||||
document-start:
|
||||
level: warning
|
||||
present: false
|
||||
line-length:
|
||||
level: warning
|
||||
max: 80
|
||||
allow-non-breakable-words: true
|
||||
allow-non-breakable-inline-mappings: true
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,73 +0,0 @@
|
|||
{
|
||||
"name": "compressed-cache-action",
|
||||
"description": "action to save/restore cache as tarballs",
|
||||
"version": "0.0.1",
|
||||
"author": "OpenVINO Developers",
|
||||
"private": true,
|
||||
"keywords": [
|
||||
"GitHub",
|
||||
"Actions",
|
||||
"JavaScript"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"main": "dist/restore/index.js",
|
||||
"scripts": {
|
||||
"bundle": "npm run format:write && npm run package",
|
||||
"ci-test": "npx jest",
|
||||
"coverage": "npx jest -- --coverage",
|
||||
"format:write": "npx prettier --write .",
|
||||
"format:check": "npx prettier --check .",
|
||||
"lint": "npx eslint . -c ./linters/.eslintrc.yml ",
|
||||
"package": "npx ncc build src/restore.js -o dist/restore && npx ncc build src/save.js -o dist/save && npx ncc build src/restore.js -o dist/restore-only && npx ncc build src/save.js -o dist/save-only && npx ncc build src/cleanup.js -o dist/cleanup-only",
|
||||
"package:watch": "npm run package -- --watch",
|
||||
"test": "npx jest",
|
||||
"all": "npm run format:write && npm run lint && npm run test && npm run coverage && npm run package"
|
||||
},
|
||||
"eslintConfig": {
|
||||
"extends": "./linters/.eslintrc.yml"
|
||||
},
|
||||
"jest": {
|
||||
"verbose": true,
|
||||
"clearMocks": true,
|
||||
"testEnvironment": "node",
|
||||
"moduleFileExtensions": [
|
||||
"js"
|
||||
],
|
||||
"testMatch": [
|
||||
"**/*.test.js"
|
||||
],
|
||||
"testPathIgnorePatterns": [
|
||||
"/node_modules/",
|
||||
"/dist/"
|
||||
],
|
||||
"coverageReporters": [
|
||||
"json-summary",
|
||||
"text",
|
||||
"lcov"
|
||||
],
|
||||
"collectCoverage": true,
|
||||
"collectCoverageFrom": [
|
||||
"./src/**"
|
||||
]
|
||||
},
|
||||
"dependencies": {
|
||||
"@actions/core": "^1.10.1",
|
||||
"fs": "^0.0.1-security",
|
||||
"tar": "^6.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/core": "^7.24.0",
|
||||
"@babel/eslint-parser": "^7.23.10",
|
||||
"@babel/preset-env": "^7.24.0",
|
||||
"@vercel/ncc": "^0.38.1",
|
||||
"babel-preset-jest": "^29.6.3",
|
||||
"eslint": "^8.57.0",
|
||||
"eslint-plugin-github": "^4.10.2",
|
||||
"eslint-plugin-jest": "^27.9.0",
|
||||
"fs": "0.0.1-security",
|
||||
"jest": "^29.7.0",
|
||||
"prettier": "^3.2.5"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
name: 'Restore Cache'
|
||||
description: 'Restore Cache artifacts to improve workflow execution time'
|
||||
author: 'OpenVINO Developers'
|
||||
|
||||
inputs:
|
||||
cache-path:
|
||||
description: 'A local path where to search and save caches'
|
||||
required: true
|
||||
path:
|
||||
description:
|
||||
'A list of files, directories, and wildcard patterns to cache and restore'
|
||||
required: true
|
||||
key:
|
||||
description: 'An explicit key for restoring and saving the cache'
|
||||
required: true
|
||||
restore-keys:
|
||||
description:
|
||||
'An ordered list of keys to use for restoring stale cache if no cache hit
|
||||
occurred for key.'
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
outputs:
|
||||
cache-hit:
|
||||
description:
|
||||
'A boolean value to indicate an exact match was found for the primary key'
|
||||
cache-file:
|
||||
description: 'Found cache file'
|
||||
|
||||
runs:
|
||||
using: 'node20'
|
||||
main: '../dist/restore-only/index.js'
|
||||
branding:
|
||||
icon: 'archive'
|
||||
color: 'gray-dark'
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
name: 'Save a cache'
|
||||
description: 'Cache to the local file system'
|
||||
author: 'OpenVINO Developers'
|
||||
|
||||
inputs:
|
||||
cache-path:
|
||||
description: 'A local path where to search and save caches'
|
||||
required: true
|
||||
path:
|
||||
description:
|
||||
'A list of files, directories, and wildcard patterns to cache and restore'
|
||||
required: true
|
||||
key:
|
||||
description: 'An explicit key for restoring and saving the cache'
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: 'node20'
|
||||
main: '../dist/save-only/index.js'
|
||||
branding:
|
||||
icon: 'archive'
|
||||
color: 'gray-dark'
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
const { cleanUp } = require('./cleanupImpl');
|
||||
|
||||
cleanUp();
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
const core = require('@actions/core');
|
||||
const fs = require('fs/promises');
|
||||
const path = require('path');
|
||||
const {
|
||||
getSortedCacheFiles,
|
||||
humanReadableFileSize,
|
||||
calculateTotalSize
|
||||
} = require('./utils');
|
||||
|
||||
// Function to remove old files if their combined size exceeds the allowed size
|
||||
async function cleanUp() {
|
||||
try {
|
||||
const cacheRemotePath = core.getInput('cache-path', { required: true });
|
||||
const key = core.getInput('key', { required: true });
|
||||
const keysRestore = core
|
||||
.getInput('restore-keys', { required: false })
|
||||
.split('\n')
|
||||
.map(s => s.replace(/^!\s+/, '!').trim())
|
||||
.filter(x => x !== '');
|
||||
const cacheSize = core.getInput('cache-size', { required: false });
|
||||
const cacheMaxSize = core.getInput('max-cache-size', { required: false });
|
||||
|
||||
// Minimum time peroid in milliseconds when the files was not useds
|
||||
const minAccessTime = 7 * 24 * 60 * 60 * 1000; // 1 week
|
||||
const currentDate = new Date();
|
||||
const minAccessDateAgo = new Date(currentDate - minAccessTime);
|
||||
|
||||
core.debug(`cache-path: ${cacheRemotePath}`);
|
||||
core.debug(`key: ${key}`);
|
||||
core.debug(`restore-keys: ${keysRestore}`);
|
||||
core.debug(`cache-size: ${cacheSize}`);
|
||||
core.debug(`max-cache-size: ${cacheMaxSize}`);
|
||||
|
||||
let keyPattern = key;
|
||||
if (keysRestore && keysRestore.length) {
|
||||
keyPattern = keysRestore.join('|');
|
||||
}
|
||||
|
||||
const files = await getSortedCacheFiles(cacheRemotePath, keyPattern);
|
||||
const minCacheSizeInBytes = cacheSize * 1024 * 1024 * 1024;
|
||||
const maxCacheSizeInBytes = cacheMaxSize * 1024 * 1024 * 1024;
|
||||
let totalSize = await calculateTotalSize(cacheRemotePath, files);
|
||||
|
||||
if (totalSize <= minCacheSizeInBytes) {
|
||||
core.info(
|
||||
`The cache storage size ${humanReadableFileSize(totalSize)} less then the allowed size ${humanReadableFileSize(minCacheSizeInBytes)}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
core.info(
|
||||
`The cache storage size ${humanReadableFileSize(totalSize)} exceeds allowed size ${humanReadableFileSize(minCacheSizeInBytes)}`
|
||||
);
|
||||
for (const file of files.reverse()) {
|
||||
const filePath = path.join(cacheRemotePath, file);
|
||||
const fileStats = await fs.stat(filePath);
|
||||
|
||||
// skipping recently used files if total cache size less then maxCacheSizeInBytes
|
||||
if (
|
||||
!fileStats.isFile() ||
|
||||
(fileStats.atime >= minAccessDateAgo && totalSize < maxCacheSizeInBytes)
|
||||
) {
|
||||
core.info(`Skipping: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
core.info(`Removing file: ${filePath}`);
|
||||
await fs.unlink(filePath);
|
||||
core.info(`${filePath} removed successfully`);
|
||||
totalSize -= fileStats.size;
|
||||
|
||||
// Exit if total size is within limit
|
||||
if (totalSize <= minCacheSizeInBytes) {
|
||||
core.info('Old cache files removed successfully');
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
core.error(`Error removing old cache files: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cleanUp
|
||||
};
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
/**
|
||||
* The entrypoint for the action.
|
||||
*/
|
||||
const { restore } = require('./restoreImpl');
|
||||
|
||||
restore();
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
const core = require('@actions/core');
|
||||
const fs = require('fs/promises');
|
||||
const path = require('path');
|
||||
const tar = require('tar');
|
||||
const os = require('os');
|
||||
|
||||
const {
|
||||
getSortedCacheFiles,
|
||||
humanReadableFileSize,
|
||||
checkFileExists
|
||||
} = require('./utils');
|
||||
|
||||
/**
|
||||
* The main function for the action.
|
||||
* @returns {Promise<void>} Resolves when the action is complete.
|
||||
*/
|
||||
async function restore() {
|
||||
try {
|
||||
const cacheRemotePath = core.getInput('cache-path', { required: true });
|
||||
const cacheLocalPath = core.getInput('path', { required: true });
|
||||
const key = core.getInput('key', { required: true });
|
||||
const keysRestore = core
|
||||
.getInput('restore-keys', { required: false })
|
||||
.split('\n')
|
||||
.map(s => s.replace(/^!\s+/, '!').trim())
|
||||
.filter(x => x !== '');
|
||||
|
||||
core.debug(`cache-path: ${cacheRemotePath}`);
|
||||
core.debug(`path: ${cacheLocalPath}`);
|
||||
core.debug(`key: ${key}`);
|
||||
core.debug(`restore-keys: ${keysRestore}`);
|
||||
|
||||
let keyPattern = key;
|
||||
if (keysRestore && keysRestore.length) {
|
||||
keyPattern = keysRestore.join('|');
|
||||
}
|
||||
|
||||
core.info(`Looking for ${keyPattern} in ${cacheRemotePath}`);
|
||||
const files = await getSortedCacheFiles(cacheRemotePath, keyPattern);
|
||||
|
||||
if (files.length) {
|
||||
const cacheFile = files[0];
|
||||
const cachePath = path.join(cacheRemotePath, cacheFile);
|
||||
const cacheStat = await fs.stat(cachePath);
|
||||
|
||||
core.info(
|
||||
`Found cache file: ${cacheFile}, size: ${humanReadableFileSize(cacheStat.size)}`
|
||||
);
|
||||
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cache-'));
|
||||
// copy file to temp dir
|
||||
await fs.copyFile(cachePath, path.join(tempDir, cacheFile));
|
||||
core.info(`${cacheFile} was copied to ${tempDir}/${cacheFile}`);
|
||||
|
||||
// extract
|
||||
if (!(await checkFileExists(cacheLocalPath))) {
|
||||
await fs.mkdir(cacheLocalPath);
|
||||
}
|
||||
core.info(`Extracting ${cacheFile} to ${cacheLocalPath}`);
|
||||
tar.x({
|
||||
file: path.join(tempDir, cacheFile),
|
||||
cwd: cacheLocalPath,
|
||||
sync: true
|
||||
});
|
||||
core.info(`Cache extracted to ${cacheLocalPath}`);
|
||||
|
||||
core.setOutput('cache-file', cacheFile);
|
||||
core.setOutput('cache-hit', true);
|
||||
} else {
|
||||
core.warning(
|
||||
`Could not found any suitable cache files in ${cacheRemotePath} with key ${key}`
|
||||
);
|
||||
core.setOutput('cache-file', '');
|
||||
core.setOutput('cache-hit', false);
|
||||
}
|
||||
} catch (error) {
|
||||
// do not fail action if cache could not be restored
|
||||
core.error(error.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
restore
|
||||
};
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
const core = require('@actions/core');
|
||||
const { save } = require('./saveImpl');
|
||||
const { cleanUp } = require('./cleanupImpl');
|
||||
|
||||
const saveAlways = core.getInput('save-always', { required: false });
|
||||
const cleanUpAlways = core.getInput('cleanup-always', { required: false });
|
||||
|
||||
if (saveAlways === 'true') {
|
||||
save();
|
||||
}
|
||||
|
||||
if (cleanUpAlways === 'true') {
|
||||
cleanUp();
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
const core = require('@actions/core');
|
||||
const tar = require('tar');
|
||||
const fs = require('fs/promises');
|
||||
const path = require('path');
|
||||
const { humanReadableFileSize, checkFileExists } = require('./utils');
|
||||
|
||||
/**
|
||||
* The main function for the action.
|
||||
* @returns {Promise<void>} Resolves when the action is complete.
|
||||
*/
|
||||
async function save() {
|
||||
try {
|
||||
const cacheRemotePath = core.getInput('cache-path', { required: true });
|
||||
const toCachePath = core.getInput('path', { required: true });
|
||||
const key = core.getInput('key', { required: true });
|
||||
|
||||
core.debug(`cache-path: ${cacheRemotePath}`);
|
||||
core.debug(`path: ${toCachePath}`);
|
||||
core.debug(`key: ${key}`);
|
||||
|
||||
if (!key) {
|
||||
core.warning(`Key ${key} is not specified.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const tarName = `${key}.cache`;
|
||||
const tarPath = path.join(cacheRemotePath, tarName);
|
||||
const tarNameTmp = `${key}.tmp`;
|
||||
const tarPathTmp = path.join(cacheRemotePath, tarNameTmp);
|
||||
|
||||
if (await checkFileExists(tarPath)) {
|
||||
core.warning(`Cache file ${tarName} already exists`);
|
||||
return;
|
||||
}
|
||||
|
||||
core.info(`Preparing cache archive ${tarName}`);
|
||||
tar.c(
|
||||
{
|
||||
gzip: true,
|
||||
file: tarName,
|
||||
cwd: toCachePath,
|
||||
sync: true
|
||||
},
|
||||
['.']
|
||||
);
|
||||
const tarStat = await fs.stat(tarName);
|
||||
core.info(
|
||||
`Created cache tarball: ${tarName}, size: ${humanReadableFileSize(tarStat.size)}`
|
||||
);
|
||||
|
||||
// remote cache directory may not be created yet
|
||||
if (!(await checkFileExists(cacheRemotePath))) {
|
||||
await fs.mkdir(cacheRemotePath, { recursive: true });
|
||||
}
|
||||
|
||||
core.info('Copying cache...');
|
||||
await fs.copyFile(tarName, tarPathTmp);
|
||||
// After copying is done, rename file
|
||||
await fs.rename(tarPathTmp, tarPath);
|
||||
core.info(`${tarName} copied to ${tarPath}`);
|
||||
|
||||
core.setOutput('cache-file', tarName);
|
||||
core.setOutput('cache-hit', true);
|
||||
} catch (error) {
|
||||
core.setFailed(error.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
save
|
||||
};
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
const core = require('@actions/core');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
async function getSortedCacheFiles(cachePath, key = '') {
|
||||
if (!(await checkFileExists(cachePath))) {
|
||||
core.warning(`${cachePath} doesn't exist`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const cachePattern = new RegExp(`^((${key}).*[.]cache)$`);
|
||||
|
||||
const filesSorted = (await fs.promises.readdir(cachePath))
|
||||
.filter(fileName => cachePattern.test(fileName))
|
||||
.map(fileName => ({
|
||||
name: fileName,
|
||||
time: fs.statSync(path.join(cachePath, fileName)).atimeMs
|
||||
}))
|
||||
.sort((a, b) => b.time - a.time)
|
||||
.map(file => file.name);
|
||||
|
||||
core.debug(
|
||||
filesSorted.map(fileName => ({
|
||||
name: fileName,
|
||||
atime: fs.statSync(path.join(cachePath, fileName)).atimeMs
|
||||
}))
|
||||
);
|
||||
return filesSorted;
|
||||
}
|
||||
|
||||
function humanReadableFileSize(sizeInBytes) {
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let id = 0;
|
||||
|
||||
while (sizeInBytes >= 1024 && id < units.length - 1) {
|
||||
sizeInBytes /= 1024;
|
||||
id++;
|
||||
}
|
||||
|
||||
return `${sizeInBytes.toFixed(2)} ${units[id]}`;
|
||||
}
|
||||
|
||||
// Function to calculate the total size of files in bytes
|
||||
async function calculateTotalSize(dir, files) {
|
||||
let totalSize = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(dir, file);
|
||||
const fileStats = await fs.promises.stat(filePath);
|
||||
|
||||
if (fileStats.isFile()) {
|
||||
totalSize += fileStats.size;
|
||||
}
|
||||
}
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
async function checkFileExists(filePath) {
|
||||
try {
|
||||
await fs.promises.access(filePath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getSortedCacheFiles,
|
||||
humanReadableFileSize,
|
||||
calculateTotalSize,
|
||||
checkFileExists
|
||||
};
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
name: 'Handle Docker images'
|
||||
description: 'Builds, tags and pushes a given Docker image when needed'
|
||||
inputs:
|
||||
images:
|
||||
description: 'Image names (registry name + namespace + base name)'
|
||||
required: true
|
||||
registry:
|
||||
description: 'Docker registry'
|
||||
required: true
|
||||
dockerfiles_root_dir:
|
||||
description: 'Path to dockerfiles root dir relative to repository root'
|
||||
required: true
|
||||
push:
|
||||
description: 'Push built images to registry'
|
||||
required: false
|
||||
default: 'true'
|
||||
changed_components:
|
||||
description: 'Components changed by a pull request'
|
||||
required: true
|
||||
|
||||
outputs:
|
||||
images:
|
||||
description: "Images to use in workflow"
|
||||
value: ${{ steps.handle_images.outputs.images }}
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Checkout head
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Checkout base
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.base_ref || github.event.merge_group.base_ref }}
|
||||
sparse-checkout: ${{ inputs.dockerfiles_root_dir }}/docker_tag
|
||||
path: base
|
||||
|
||||
- name: Install Python dependencies
|
||||
uses: py-actions/py-dependency-install@v4
|
||||
with:
|
||||
path: "${{ github.action_path }}/requirements.txt"
|
||||
update-setuptools: "false"
|
||||
update-wheel: "false"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Handle docker images
|
||||
id: handle_images
|
||||
shell: bash
|
||||
run: |
|
||||
images=$(echo "${{ inputs.images }}" | tr '\n' ',' | sed 's/,*$//')
|
||||
pr="${{ github.event.pull_request.number }}"
|
||||
|
||||
python3 .github/actions/handle_docker/get_images_to_build.py \
|
||||
-d "${{ inputs.dockerfiles_root_dir }}" \
|
||||
-r "${{ inputs.registry }}" \
|
||||
--images "$images" \
|
||||
--head_tag_file "${{ inputs.dockerfiles_root_dir }}/docker_tag" \
|
||||
--base_tag_file "base/${{ inputs.dockerfiles_root_dir }}/docker_tag" \
|
||||
--docker_env_changed "${{ fromJSON(inputs.changed_components).docker_env }}" \
|
||||
--dockerfiles_changed "${{ fromJSON(inputs.changed_components).dockerfiles }}" \
|
||||
--docker_builder "${{ steps.buildx.outputs.name}}" \
|
||||
--repo "${{ github.repository }}" \
|
||||
--ref_name "${{ github.ref_name }}" \
|
||||
$([[ -n $pr ]] && echo "--pr $pr" || echo '-s ${{ github.sha }}') \
|
||||
$([[ -n "${{ inputs.push }}" ]] && echo "--push" || echo '')
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
from distutils.util import strtobool
|
||||
from helpers import *
|
||||
from images_api import *
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Returns list of Docker images to build for a given workflow')
|
||||
parser.add_argument('-i', '--images', required=True, help='Comma-separated docker images')
|
||||
parser.add_argument('-d', '--dockerfiles_root', required=True, help='Path to dockerfiles')
|
||||
parser.add_argument('-r', '--registry', required=True, help='Docker registry name')
|
||||
parser.add_argument('-s', '--commit', required=False, help='Commit SHA. If not set, --pr is used')
|
||||
parser.add_argument('-b', '--docker_builder', required=False, help='Docker buildx builder name')
|
||||
parser.add_argument('--pr', type=int, required=False, help='PR number, if event is pull_request')
|
||||
parser.add_argument('--head_tag_file', default='.github/dockerfiles/docker_tag', help='Head docker tag file path')
|
||||
parser.add_argument('--base_tag_file', default=None, required=False, help='Base docker tag file path')
|
||||
parser.add_argument('--ref_name', required=False, default='', help='GitHub ref name')
|
||||
parser.add_argument('--repo', default='openvinotoolkit/openvino', help='GitHub repository')
|
||||
parser.add_argument('--docker_env_changed', type=lambda x: bool(strtobool(x)), default=True,
|
||||
help='Whether PR changes docker env')
|
||||
parser.add_argument('--dockerfiles_changed', type=lambda x: bool(strtobool(x)), default=True,
|
||||
help='Whether PR changes dockerfiles')
|
||||
parser.add_argument('--action_path', default='.github/actions/handle_docker', help='Path to this GitHub action')
|
||||
parser.add_argument('--push', action='store_true', required=False, help='Whether to push images to registry')
|
||||
parser.add_argument('--dry_run', action='store_true', required=False, help='Dry run')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
init_logger()
|
||||
logger = logging.getLogger(__name__)
|
||||
args = parse_args()
|
||||
for arg, value in sorted(vars(args).items()):
|
||||
logger.info(f"Argument {arg}: {value}")
|
||||
|
||||
head_tag = Path(args.head_tag_file).read_text().strip()
|
||||
|
||||
base_tag_exists = args.base_tag_file and Path(args.base_tag_file).exists()
|
||||
base_tag = Path(args.base_tag_file).read_text().strip() if base_tag_exists else None
|
||||
|
||||
all_dockerfiles = Path(args.dockerfiles_root).rglob('**/*/Dockerfile')
|
||||
|
||||
images = ImagesHandler(args.dry_run)
|
||||
for image in all_dockerfiles:
|
||||
images.add_from_dockerfile(image, args.dockerfiles_root, args.registry, head_tag, base_tag)
|
||||
|
||||
requested_images = set(args.images.split(','))
|
||||
skip_workflow = False
|
||||
missing_only = False
|
||||
|
||||
merge_queue_target_branch = next(iter(re.findall(f'^gh-readonly-queue/(.*)/', args.ref_name)), None)
|
||||
|
||||
if args.pr:
|
||||
environment_affected = args.docker_env_changed or args.dockerfiles_changed
|
||||
if environment_affected:
|
||||
expected_tag = f'pr-{args.pr}'
|
||||
|
||||
if head_tag != expected_tag:
|
||||
logger.error(f"Please update docker tag in {args.head_tag_file} to {expected_tag}")
|
||||
sys.exit(1)
|
||||
|
||||
elif merge_queue_target_branch:
|
||||
environment_affected = head_tag != base_tag
|
||||
if environment_affected:
|
||||
logger.info(f"Environment is affected by PR(s) in merge group")
|
||||
else:
|
||||
environment_affected = False
|
||||
|
||||
if environment_affected:
|
||||
changeset = get_changeset(args.repo, args.pr, merge_queue_target_branch, args.commit)
|
||||
changed_dockerfiles = [p for p in changeset if p.startswith(args.dockerfiles_root) and p.endswith('Dockerfile')]
|
||||
|
||||
if args.docker_env_changed:
|
||||
logger.info(f"Common docker environment is modified, will build all requested images")
|
||||
changed_images = requested_images
|
||||
else:
|
||||
logger.info(f"Common docker environment is not modified, will build only changed and missing images")
|
||||
changed_images = set([name_from_dockerfile(d, args.dockerfiles_root) for d in changed_dockerfiles])
|
||||
|
||||
unchanged_images = requested_images - changed_images
|
||||
unchanged_with_no_base = images.get_missing(unchanged_images, base=True)
|
||||
|
||||
if unchanged_with_no_base:
|
||||
logger.info("The following images were unchanged, but will be built anyway since the base for them "
|
||||
f"is missing in registry: {unchanged_with_no_base}")
|
||||
|
||||
images_to_tag = unchanged_images.difference(unchanged_with_no_base)
|
||||
images_to_build = requested_images.intersection(changed_images).union(unchanged_with_no_base)
|
||||
|
||||
only_dockerfiles_changed = len(changeset) == len(changed_dockerfiles)
|
||||
if only_dockerfiles_changed and not images_to_build:
|
||||
skip_workflow = True
|
||||
else:
|
||||
logger.info(f"Environment is not affected, will build only missing images, if any")
|
||||
images_to_build = requested_images
|
||||
images_to_tag = []
|
||||
missing_only = True
|
||||
|
||||
if not images_to_build:
|
||||
logger.info(f"No images to build, will return the list of pre-built images with a new tag")
|
||||
|
||||
built_images = images.build(images_to_build, missing_only, args.push, args.docker_builder)
|
||||
if not built_images:
|
||||
logger.info(f"No images were built, a new tag will be applied to a pre-built base image if needed")
|
||||
|
||||
# When a custom builder is used, it allows to push the image automatically once built. Otherwise, pushing manually
|
||||
if args.push and not args.docker_builder:
|
||||
images.push(images_to_build, missing_only)
|
||||
|
||||
if environment_affected and base_tag:
|
||||
images.tag(images_to_tag)
|
||||
|
||||
images_output = images_to_output(images.get(requested_images))
|
||||
set_github_output("images", json.dumps(images_output))
|
||||
|
||||
if skip_workflow:
|
||||
logger.info(f"Docker image changes are irrelevant for current workflow, workflow may be skipped")
|
||||
set_github_output("skip_workflow", str(skip_workflow))
|
||||
|
||||
|
||||
main()
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from ghapi.all import GhApi
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def init_logger():
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format='%(asctime)s %(name)-15s %(levelname)-8s %(message)s',
|
||||
datefmt='%m-%d-%Y %H:%M:%S')
|
||||
|
||||
|
||||
def set_github_output(name: str, value: str, github_output_var_name: str = 'GITHUB_OUTPUT'):
|
||||
"""Sets output variable for a GitHub Action"""
|
||||
logger = logging.getLogger(__name__)
|
||||
# In an environment variable "GITHUB_OUTPUT" GHA stores path to a file to write outputs to
|
||||
with open(os.environ.get(github_output_var_name), 'a+') as file:
|
||||
logger.info(f"Add {name}={value} to {github_output_var_name}")
|
||||
print(f'{name}={value}', file=file)
|
||||
|
||||
|
||||
def images_to_output(images: list):
|
||||
images_output = {}
|
||||
for image in images:
|
||||
image_name, os_name = image.name.split('/', 1)
|
||||
if image_name not in images_output:
|
||||
images_output[image_name] = {}
|
||||
|
||||
images_output[image_name][os_name] = image.ref()
|
||||
|
||||
return images_output
|
||||
|
||||
|
||||
def get_changeset(repo: str, pr: str, target_branch: str, commit_sha: str):
|
||||
"""Returns changeset either from PR or commit"""
|
||||
owner, repository = repo.split('/')
|
||||
gh_api = GhApi(owner=owner, repo=repository, token=os.getenv("GITHUB_TOKEN"))
|
||||
if pr:
|
||||
changed_files = gh_api.pulls.list_files(pr)
|
||||
elif target_branch:
|
||||
target_branch_head_commit = gh_api.repos.get_branch(target_branch).commit.sha
|
||||
changed_files = gh_api.repos.compare_commits(f'{target_branch_head_commit}...{commit_sha}').get('files', [])
|
||||
else:
|
||||
raise ValueError(f'Either "pr" or "target_branch" parameter must be non-empty')
|
||||
return set([f.filename for f in changed_files])
|
||||
|
||||
|
||||
def run(cmd: str, dry_run: bool = False, fail_on_error: bool = True):
|
||||
logger = logging.getLogger('run')
|
||||
logger.info(cmd)
|
||||
|
||||
if dry_run:
|
||||
return 0, ''
|
||||
|
||||
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as proc:
|
||||
for line in proc.stdout:
|
||||
logger.info(line.strip())
|
||||
|
||||
proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
msg = f"Command '{cmd}' returned non-zero exit status {proc.returncode}"
|
||||
if fail_on_error:
|
||||
raise RuntimeError(msg)
|
||||
|
||||
logger.warning(msg)
|
||||
return proc.returncode
|
||||
|
||||
|
||||
def name_from_dockerfile(dockerfile: str | Path, dockerfiles_root: str | Path) -> str:
|
||||
image_name = str(Path(dockerfile).relative_to(dockerfiles_root).parent.as_posix())
|
||||
return image_name
|
||||
|
|
@ -1,137 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from helpers import run, name_from_dockerfile
|
||||
|
||||
|
||||
class Image:
|
||||
def __init__(self, name: str, dockerfile: Path, registry: str):
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
self.name = name
|
||||
self.dockerfile = dockerfile
|
||||
self.registry = registry
|
||||
self.tag = 'latest'
|
||||
self.base_tag = None
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
def __eq__(self, img):
|
||||
return img.name == self.name if img else False
|
||||
|
||||
def with_tag(self, tag: str):
|
||||
self.tag = tag
|
||||
return self
|
||||
|
||||
def with_base_tag(self, tag: str):
|
||||
self.base_tag = tag
|
||||
return self
|
||||
|
||||
def ref(self):
|
||||
return f"{self.registry}/{self.name}:{self.tag}"
|
||||
|
||||
def base_ref(self):
|
||||
if not self.base_tag:
|
||||
return None
|
||||
return f"{self.registry}/{self.name}:{self.base_tag}"
|
||||
|
||||
def push(self, dry: bool = False):
|
||||
cmd = f"docker push {self.ref()} "
|
||||
run(cmd, dry)
|
||||
|
||||
def build(self, dry: bool = False, push: bool = True, docker_builder: str = None, import_cache: bool = True,
|
||||
export_cache: bool = True):
|
||||
cache_cmd = ""
|
||||
if import_cache:
|
||||
cache_cmd += f"--cache-from type=registry,ref={self.ref()}-cache "
|
||||
if self.base_tag:
|
||||
cache_cmd += f"--cache-from type=registry,ref={self.base_ref()}-cache "
|
||||
|
||||
if export_cache:
|
||||
cache_cmd += f"--cache-to type=registry,ref={self.ref()}-cache,mode=max "
|
||||
|
||||
build_cmd = f"docker buildx build --builder={docker_builder}" if docker_builder else "docker build"
|
||||
push_cmd = f"--push" if push else ""
|
||||
|
||||
cmd = f"{build_cmd} " \
|
||||
f"--file {self.dockerfile} " \
|
||||
f"--tag {self.ref()} " \
|
||||
f"{cache_cmd} " \
|
||||
f"{push_cmd} " \
|
||||
"."
|
||||
|
||||
run(cmd, dry)
|
||||
|
||||
def tag_base(self, dry: bool = False):
|
||||
if not self.base_tag:
|
||||
raise AttributeError("Tag for base image is not specified")
|
||||
|
||||
cmd = f"docker buildx imagetools create -t {self.ref()} {self.base_ref()}"
|
||||
|
||||
run(cmd, dry)
|
||||
|
||||
def is_missing(self, dry: bool = False, base: bool = False) -> bool:
|
||||
image = self.base_ref() if base else self.ref()
|
||||
if base and not image:
|
||||
self.logger.warning(f"Base ref for image {self.ref()} is missing")
|
||||
return True
|
||||
|
||||
cmd = f"docker manifest inspect {image}"
|
||||
is_missing = False
|
||||
|
||||
self.logger.info(cmd)
|
||||
if not dry:
|
||||
try:
|
||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True, text=True)
|
||||
except subprocess.CalledProcessError:
|
||||
self.logger.warning(f"{image} is missing in registry")
|
||||
is_missing = True
|
||||
|
||||
return is_missing
|
||||
|
||||
|
||||
# Making it a class, so it's a little easier to switch to a tree structure for building inherited images if we want
|
||||
class ImagesHandler:
|
||||
def __init__(self, dry_run: bool = False):
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
self.images = dict()
|
||||
self.dry_run = dry_run
|
||||
|
||||
def add_from_dockerfile(self, dockerfile: str | Path, dockerfiles_root: str | Path, registry: str, tag: str,
|
||||
base_tag: str = None):
|
||||
image_name = name_from_dockerfile(dockerfile, dockerfiles_root)
|
||||
image = Image(image_name, Path(dockerfile), registry).with_tag(tag).with_base_tag(base_tag)
|
||||
self.add(image)
|
||||
|
||||
def add(self, image: Image):
|
||||
self.images[image.name] = image
|
||||
|
||||
def get(self, image_names: Iterable = None) -> list:
|
||||
images = [self.images[name] for name in image_names] if image_names is not None else self.images.values()
|
||||
return images
|
||||
|
||||
def get_missing(self, image_names: Iterable = None, base: bool = False) -> list:
|
||||
missing_images = [image.name for image in self.get(image_names) if image.is_missing(self.dry_run, base)]
|
||||
return missing_images
|
||||
|
||||
def build(self, image_names: Iterable = None, missing_only: bool = False, push: bool = True, builder: str = None,
|
||||
import_cache: bool = True, export_cache: bool = True):
|
||||
to_build = self.get(self.get_missing(image_names)) if missing_only else self.get(image_names)
|
||||
for image in to_build:
|
||||
image.build(self.dry_run, push, builder, import_cache, export_cache)
|
||||
return to_build
|
||||
|
||||
def push(self, image_names: Iterable = None, missing_only: bool = False):
|
||||
to_push = self.get(self.get_missing(image_names)) if missing_only else self.get(image_names)
|
||||
for image in to_push:
|
||||
image.push(self.dry_run)
|
||||
|
||||
def tag(self, image_names: Iterable = None):
|
||||
for image in self.get(image_names):
|
||||
image.tag_base(self.dry_run)
|
||||
|
|
@ -1 +0,0 @@
|
|||
ghapi~=1.0.5
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
name: 'Setup Python and pip cache'
|
||||
description: 'Setups Python with the provided version and sets up the pip cache'
|
||||
inputs:
|
||||
version:
|
||||
description: 'Python version to install'
|
||||
required: true
|
||||
pip-cache-path:
|
||||
description: 'Path on share where the pip cache is stored'
|
||||
required: false
|
||||
should-setup-pip-paths:
|
||||
description: 'If the action should setup `PIP_CACHE_DIR` & `PIP_INSTALL_PATH` env variables'
|
||||
required: false
|
||||
default: 'false'
|
||||
self-hosted-runner:
|
||||
description: 'If the runner is self-hosted'
|
||||
required: false
|
||||
default: 'true'
|
||||
show-cache-info:
|
||||
description: 'If the action should show the share space occupied by cache'
|
||||
required: false
|
||||
default: 'false'
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
|
||||
- name: Check if Python is already installed (Linux)
|
||||
if: ${{ runner.os == 'Linux' }}
|
||||
shell: bash
|
||||
id: check_python
|
||||
run: |
|
||||
PYTHON_INSTALLED=$(python${{ inputs.version }} -V) || true
|
||||
if [[ $PYTHON_INSTALLED ]]; then
|
||||
echo "installed=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "installed=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- if: ${{ runner.os == 'Linux' && inputs.self-hosted-runner == 'true' && steps.check_python.outputs.installed == 'false' }}
|
||||
name: Install 'actions/setup-python@v4' dependencies
|
||||
shell: bash
|
||||
run: apt-get update && apt-get install -y ca-certificates software-properties-common gpg-agent tzdata
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
|
||||
TZ: "Europe/London" # to prevent tzdata from waiting user input
|
||||
|
||||
- if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' && steps.check_python.outputs.installed == 'false' }}
|
||||
name: Setup sudo and python3
|
||||
shell: bash
|
||||
run: apt-get update && apt-get install -y sudo python3 # Needed for the deadsnakes action
|
||||
|
||||
- if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' && steps.check_python.outputs.installed == 'false' }}
|
||||
name: Setup Python ${{ inputs.version }}
|
||||
uses: akashchi/deadsnakes-action@92417281055a5878a0450f240a5b95883eb2d7e2
|
||||
with:
|
||||
python-version: ${{ inputs.version }}
|
||||
|
||||
- if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false' ) }}
|
||||
name: Setup Python ${{ inputs.version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.version }}
|
||||
env:
|
||||
PIP_CACHE_DIR: ${{ inputs.self-hosted-runner == 'true' && inputs.pip-cache-path || '' }}
|
||||
|
||||
- if: ${{ inputs.should-setup-pip-paths == 'true' }}
|
||||
name: Setup pip variables (cache and install path)
|
||||
shell: bash
|
||||
run: |
|
||||
PIP_VER=$(python3 -c "import pip; print(pip.__version__)")
|
||||
echo "Using pip version: ${PIP_VER}"
|
||||
echo "PIP_CACHE_DIR=${{ inputs.pip-cache-path }}/${PIP_VER}" >> $GITHUB_ENV
|
||||
echo "PIP_INSTALL_PATH=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" >> $GITHUB_ENV
|
||||
|
||||
- if: ${{ inputs.show-cache-info == 'true' }}
|
||||
name: Get pip cache info
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Cache size: "
|
||||
du -h -d2 ${{ env.PIP_CACHE_DIR }}
|
||||
echo "Cache info: "
|
||||
python3 -m pip cache info
|
||||
continue-on-error: true
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
name: "Smart CI action"
|
||||
description: "Returns product components affected by PR or commit"
|
||||
inputs:
|
||||
repository:
|
||||
description: "GitHub repository"
|
||||
required: true
|
||||
repo_token:
|
||||
description: "Token for access to GitHub repository"
|
||||
required: true
|
||||
pr:
|
||||
description: "GitHub PR number. If not set - commit is used"
|
||||
required: false
|
||||
commit_sha:
|
||||
description: "GitHub commit hash. Used if no PR number is set"
|
||||
required: false
|
||||
ref_name:
|
||||
description: "GitHub ref name"
|
||||
required: false
|
||||
component_pattern:
|
||||
description: "Pattern to extract component name from PR label. If not set, any label is considered a component name"
|
||||
required: false
|
||||
labeler_check_name:
|
||||
description: "Name of the labeler check"
|
||||
required: false
|
||||
default: "triage"
|
||||
components_config:
|
||||
description: "Path to components configuration file"
|
||||
required: false
|
||||
default: ".github/components.yml"
|
||||
components_config_schema:
|
||||
description: "Path to the schema file for components configuration"
|
||||
required: false
|
||||
default: ".github/actions/smart-ci/components_schema.yml"
|
||||
labeler_config:
|
||||
description: "Path to labeler configuration file"
|
||||
required: false
|
||||
default: ".github/labeler.yml"
|
||||
skip_when_only_listed_labels_set:
|
||||
description: "Comma-separated list of labels. If PR has only these labels set,
|
||||
return indicator that CI can be skipped"
|
||||
required: false
|
||||
skip_when_only_listed_files_changed:
|
||||
description: "Comma-separated list of patterns (fnmatch-style). If PR has only matching files changed,
|
||||
return indicator that CI can be skipped"
|
||||
required: false
|
||||
|
||||
outputs:
|
||||
all_components:
|
||||
description: "All components listed in configuration"
|
||||
value: ${{ steps.smart_ci.outputs.all_components }}
|
||||
affected_components:
|
||||
description: "Affected components to run validation for and their validation scope"
|
||||
value: ${{ steps.smart_ci.outputs.affected_components }}
|
||||
changed_components:
|
||||
description: "Actually changed components (for push events everything is marked as changed)"
|
||||
value: ${{ steps.smart_ci.outputs.changed_components }}
|
||||
skip_workflow:
|
||||
description: "Whether the workflow should be run with Smart CI rules applied or skipped completely"
|
||||
value: ${{ steps.smart_ci.outputs.skip_workflow }}
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Wait for labeler to finish
|
||||
uses: lewagon/wait-on-check-action@v1.3.1
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: ${{ inputs.labeler_check_name }}
|
||||
repo-token: ${{ inputs.repo_token }}
|
||||
wait-interval: 10
|
||||
|
||||
- name: checkout components file
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: .github/components.yml
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: Install Python dependencies
|
||||
uses: py-actions/py-dependency-install@v4
|
||||
with:
|
||||
path: "${{ github.action_path }}/requirements.txt"
|
||||
update-setuptools: "false"
|
||||
update-wheel: "false"
|
||||
|
||||
- name: Test functionality
|
||||
run: |
|
||||
python ${{ github.action_path }}/smart_ci_test.py
|
||||
shell: bash
|
||||
|
||||
- name: Smart CI
|
||||
id: smart_ci
|
||||
run: |
|
||||
python ${{ github.action_path }}/smart_ci.py \
|
||||
$([[ -n "${{ inputs.pr }}" ]] && echo '--pr ${{ inputs.pr }}' || echo '-s ${{ inputs.commit_sha }}') \
|
||||
-r ${{ inputs.repository }} \
|
||||
-f "${{ inputs.ref_name }}" \
|
||||
-p "${{ inputs.component_pattern }}" \
|
||||
-c "${{ inputs.components_config }}" \
|
||||
-m "${{ inputs.components_config_schema }}" \
|
||||
-l "${{ inputs.labeler_config }}" \
|
||||
--skip-when-only-listed-labels-set "${{ inputs.skip_when_only_listed_labels_set }}" \
|
||||
--skip-when-only-listed-files-changed "${{ inputs.skip_when_only_listed_files_changed }}"
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ inputs.repo_token }}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# YAML schema for Smart CI configuration file components.yml (see https://json-schema.org)
|
||||
|
||||
definitions:
|
||||
component_name:
|
||||
type: string
|
||||
pattern: "^[a-zA-Z_][a-zA-Z0-9_]*$"
|
||||
|
||||
component_data:
|
||||
type: object # dict
|
||||
additionalProperties: false
|
||||
properties:
|
||||
cmake:
|
||||
type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
'$ref': '#/definitions/component_name'
|
||||
|
||||
revalidate:
|
||||
oneOf:
|
||||
- type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
'$ref': '#/definitions/component_name'
|
||||
- type: string
|
||||
enum: ['all']
|
||||
|
||||
build:
|
||||
oneOf:
|
||||
- type: array
|
||||
uniqueItems: true
|
||||
items:
|
||||
'$ref': '#/definitions/component_name'
|
||||
- type: string
|
||||
enum: ['all']
|
||||
|
||||
propertyNames: # Validates component names
|
||||
'$ref': '#/definitions/component_name'
|
||||
patternProperties:
|
||||
".*": # Component (name validated via propertyNames)
|
||||
'$ref': '#/definitions/component_data'
|
||||
additionalProperties: false
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
ghapi~=1.0.4
|
||||
pyyaml~=6.0.1
|
||||
jsonschema~=4.19.1
|
||||
|
|
@ -1,258 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
import yaml
|
||||
import json
|
||||
import jsonschema
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from ghapi.all import GhApi
|
||||
from fnmatch import fnmatch
|
||||
|
||||
|
||||
class ComponentConfig:
|
||||
FullScope = {'build', 'test'}
|
||||
ScopeKeys = {'build', 'revalidate'}
|
||||
|
||||
def __init__(self, config: dict, schema: dict, all_possible_components: set):
|
||||
self.config = config
|
||||
self.log = logging.getLogger(self.__class__.__name__)
|
||||
self.all_defined_components = set(self.config.keys()) # already defined in components.yml
|
||||
# can be added to components.yml (based on labeler.yml)
|
||||
self.all_possible_components = all_possible_components.union(self.all_defined_components)
|
||||
self.validate(schema)
|
||||
|
||||
def validate(self, schema: dict) -> None:
|
||||
"""Validates syntax of configuration file"""
|
||||
jsonschema.validate(self.config, schema)
|
||||
|
||||
for component_name, data in self.config.items():
|
||||
dependent_components = set()
|
||||
for key in self.ScopeKeys:
|
||||
scope = data.get(key)
|
||||
dependent_components = dependent_components.union(set(scope) if isinstance(scope, list) else set())
|
||||
|
||||
invalid_dependents = dependent_components.difference(self.all_possible_components)
|
||||
if invalid_dependents:
|
||||
error_msg = f"dependent components of {component_name} are invalid: " \
|
||||
f"{invalid_dependents} are not listed in components config: {self.all_possible_components}"
|
||||
raise jsonschema.exceptions.ValidationError(error_msg)
|
||||
|
||||
def get_affected_components(self, changed_components_names: set) -> dict:
|
||||
"""Returns changed components, their dependencies and validation scope for them"""
|
||||
affected_components = dict()
|
||||
|
||||
# If some changed components were not defined in config or no changed components detected at all,
|
||||
# run full scope for everything (just in case)
|
||||
changed_not_defined_components = changed_components_names.difference(self.all_defined_components)
|
||||
if not changed_components_names or changed_not_defined_components:
|
||||
self.log.info(f"Changed components {changed_not_defined_components} are not defined in smart ci config, "
|
||||
"run full scope")
|
||||
affected_components.update({name: self.FullScope for name in self.all_possible_components})
|
||||
return affected_components
|
||||
|
||||
# Else check changed components' dependencies and add them to affected
|
||||
for name in changed_components_names:
|
||||
component_scopes = {k: v for k, v in self.config.get(name, dict()).items() if k in self.ScopeKeys}
|
||||
for key, dependents in component_scopes.items():
|
||||
if dependents == 'all':
|
||||
dependents = self.all_possible_components
|
||||
for dep_name in dependents:
|
||||
affected_components[dep_name] = affected_components.get(dep_name, set())
|
||||
scope = self.FullScope if key == 'revalidate' else {key}
|
||||
affected_components[dep_name] = affected_components[dep_name].union(scope)
|
||||
|
||||
if not component_scopes:
|
||||
self.log.info(f"Changed component '{name}' doesn't have {self.ScopeKeys} keys in components config. "
|
||||
f"Assuming that it affects everything, the whole scope will be started")
|
||||
for dep_name in self.all_possible_components:
|
||||
affected_components[dep_name] = self.FullScope
|
||||
|
||||
# If the component was explicitly changed, run full scope for it
|
||||
affected_components.update({name: self.FullScope for name in changed_components_names})
|
||||
self.log.info(f"Changed components with dependencies: {affected_components}")
|
||||
|
||||
# For non-affected components that are not defined in config - run full scope
|
||||
affected_components.update({name: self.FullScope for name in self.all_possible_components
|
||||
if name not in self.all_defined_components})
|
||||
|
||||
return affected_components
|
||||
|
||||
def get_static_data(self, components_names: set, data_key: str, default: str = None) -> dict:
|
||||
"""Returns requested generic static data defined for each component"""
|
||||
data = {name: self.config[name].get(data_key, default) for name in components_names}
|
||||
return data
|
||||
|
||||
|
||||
def component_name_from_label(label: str, component_pattern: str = None) -> str:
|
||||
"""Extracts component name from label"""
|
||||
component = label
|
||||
if component_pattern:
|
||||
matches = re.findall(component_pattern, label)
|
||||
component = matches[0] if matches else None
|
||||
component = component.replace(' ', '_') if component else None
|
||||
return component
|
||||
|
||||
|
||||
def get_changed_component_names(pr, all_possible_components: set, component_pattern: str = None) -> set:
|
||||
"""Returns component names changed in a given PR"""
|
||||
components = set()
|
||||
for label in pr.labels:
|
||||
component = component_name_from_label(label.name, component_pattern)
|
||||
if component:
|
||||
components.add(component)
|
||||
elif label.name in all_possible_components:
|
||||
# Allow any labels defined explicitly in labeler config as components
|
||||
# (predefined labels, such as "do not merge", are still ignored)
|
||||
components.add(label.name)
|
||||
|
||||
return components
|
||||
|
||||
|
||||
def get_changeset(gh_api, pr, target_branch, commit_sha):
|
||||
"""Returns changeset either from PR or commit"""
|
||||
if pr:
|
||||
return gh_api.pulls.list_files(pr)
|
||||
if target_branch:
|
||||
target_branch_head_commit = gh_api.repos.get_branch(target_branch).commit.sha
|
||||
# In merge-queue branch all commits between head of target branch and head of current branch (commit_sha)
|
||||
# contain changes added to queue earlier to be validated together. Getting all of them + changes from
|
||||
# commit_sha below
|
||||
changed_files = gh_api.repos.compare_commits(f'{target_branch_head_commit}...{commit_sha}').get('files', [])
|
||||
return changed_files
|
||||
raise ValueError(f'Either "pr" or "target_branch" parameter must be non-empty')
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Returns product components changed in a given PR or commit')
|
||||
parser.add_argument('--pr', type=int, required=False, help='PR number. If not set, --commit is used')
|
||||
parser.add_argument('-s', '--commit-sha', required=False, help='Commit SHA. If not set, --pr is used')
|
||||
parser.add_argument('-r', '--repo', help='GitHub repository')
|
||||
parser.add_argument('-f', '--ref_name', required=False, help='GitHub ref name')
|
||||
parser.add_argument('-p', '--pattern', default=None, help='Pattern to extract component name from PR label. '
|
||||
'If not set, any label is considered a component name')
|
||||
parser.add_argument('-c', '--components-config', default='.github/components.yml',
|
||||
help='Path to config file with info about dependencies between components')
|
||||
parser.add_argument('-m', '--components-config-schema', default='.github/actions/smart-ci/components_schema.yml',
|
||||
help='Path to the schema file for components config')
|
||||
parser.add_argument('-l', '--labeler-config', default='.github/labeler.yml',
|
||||
help='Path to PR labeler config file')
|
||||
parser.add_argument('--skip-when-only-listed-labels-set',
|
||||
help="Comma-separated list of labels. If PR has only these labels set, "
|
||||
"return indicator that CI can be skipped")
|
||||
parser.add_argument('--skip-when-only-listed-files-changed',
|
||||
help="Comma-separated list of patterns (fnmatch-style). If PR has only matching files changed, "
|
||||
"return indicator that CI can be skipped")
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def init_logger():
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format='%(asctime)s %(name)-15s %(levelname)-8s %(message)s',
|
||||
datefmt='%m-%d-%Y %H:%M:%S')
|
||||
|
||||
|
||||
def set_github_output(name: str, value: str, github_output_var_name: str = 'GITHUB_OUTPUT'):
|
||||
"""Sets output variable for a GitHub Action"""
|
||||
logger = logging.getLogger(__name__)
|
||||
# In an environment variable "GITHUB_OUTPUT" GHA stores path to a file to write outputs to
|
||||
with open(os.environ.get(github_output_var_name), 'a+') as file:
|
||||
logger.info(f"Add {name}={value} to {github_output_var_name}")
|
||||
print(f'{name}={value}', file=file)
|
||||
|
||||
|
||||
def main():
|
||||
init_logger()
|
||||
logger = logging.getLogger(__name__)
|
||||
args = parse_args()
|
||||
for arg, value in sorted(vars(args).items()):
|
||||
logger.info(f"Argument {arg}: {value}")
|
||||
|
||||
with open(Path(args.components_config), 'r') as config:
|
||||
components_config = yaml.safe_load(config)
|
||||
|
||||
owner, repository = args.repo.split('/')
|
||||
gh_api = GhApi(owner=owner, repo=repository, token=os.getenv("GITHUB_TOKEN"))
|
||||
pr = gh_api.pulls.get(args.pr) if args.pr else None
|
||||
|
||||
with open(Path(args.components_config_schema), 'r') as schema_file:
|
||||
schema = yaml.safe_load(schema_file)
|
||||
|
||||
with open(Path(args.labeler_config), 'r') as labeler_file:
|
||||
labeler_config = yaml.safe_load(labeler_file)
|
||||
|
||||
all_possible_components = set()
|
||||
for label in labeler_config.keys():
|
||||
component_name = component_name_from_label(label, args.pattern)
|
||||
all_possible_components.add(component_name if component_name else label)
|
||||
|
||||
run_full_scope = False
|
||||
# For now, we don't want to apply smart ci rules for post-commits
|
||||
is_postcommit = not pr
|
||||
|
||||
merge_queue_prefix = 'gh-readonly-queue/'
|
||||
is_merge_queue = args.ref_name.startswith(merge_queue_prefix)
|
||||
merge_queue_target_branch = re.findall(f'^{merge_queue_prefix}(.*)/', args.ref_name)[0] if is_merge_queue else None
|
||||
|
||||
if is_merge_queue:
|
||||
logger.info(f"The run is a merge-queue run, executing full validation scope for all components, if "
|
||||
f"not all queued changes match patterns in 'skip-when-only-listed-files-changed'")
|
||||
run_full_scope = True
|
||||
elif is_postcommit:
|
||||
logger.info(f"The run is a post-commit run, executing full validation scope for all components")
|
||||
run_full_scope = True
|
||||
else:
|
||||
no_match_files_changed = 'no-match-files' in [label.name for label in pr.labels]
|
||||
if no_match_files_changed:
|
||||
logger.info(f"There are changed files that don't match any pattern in labeler config, "
|
||||
f"executing full validation scope for all components")
|
||||
run_full_scope = True
|
||||
|
||||
# In post-commits - validate all components regardless of changeset
|
||||
# In pre-commits - validate only changed components with their dependencies
|
||||
all_defined_components = components_config.keys()
|
||||
changed_by_pr = get_changed_component_names(pr, all_possible_components, args.pattern) if pr else None
|
||||
changed_component_names = set(all_defined_components) if run_full_scope else changed_by_pr
|
||||
|
||||
logger.info(f"changed_component_names: {changed_component_names}")
|
||||
|
||||
cfg = ComponentConfig(components_config, schema, all_possible_components)
|
||||
affected_components = cfg.get_affected_components(changed_component_names)
|
||||
|
||||
skip_workflow = False
|
||||
if is_merge_queue or (args.pr and not run_full_scope):
|
||||
if args.skip_when_only_listed_labels_set:
|
||||
excepted_labels = set(args.skip_when_only_listed_labels_set.split(','))
|
||||
excepted_labels_only = changed_component_names - excepted_labels == set()
|
||||
skip_workflow = excepted_labels_only
|
||||
|
||||
if not skip_workflow and args.skip_when_only_listed_files_changed:
|
||||
# To avoid spending extra API requests running step below only if necessary
|
||||
changed_files = get_changeset(gh_api, args.pr, merge_queue_target_branch, args.commit_sha)
|
||||
patterns = set(args.skip_when_only_listed_files_changed.split(','))
|
||||
|
||||
matched_files_only = all(any(fnmatch(f.filename, pattern) for pattern in patterns) for f in changed_files)
|
||||
logger.debug(f"matched files only: {matched_files_only}")
|
||||
skip_workflow = matched_files_only
|
||||
|
||||
if skip_workflow:
|
||||
logger.info(f"All changes are marked for skip, workflow may be skipped")
|
||||
set_github_output("skip_workflow", str(skip_workflow))
|
||||
|
||||
# Syntactic sugar for easier use in GHA pipeline
|
||||
affected_components_output = {name: {s: True for s in scope} for name, scope in affected_components.items()}
|
||||
set_github_output("affected_components", json.dumps(affected_components_output))
|
||||
|
||||
# Components actually changed by a pull request are marked as True (if event is PR;
|
||||
# otherwise all components considered changed).
|
||||
changed_components_output = {name: True if not pr or name in changed_by_pr else False
|
||||
for name in all_possible_components}
|
||||
set_github_output("changed_components", json.dumps(changed_components_output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import unittest
|
||||
from smart_ci import ComponentConfig
|
||||
|
||||
log = logging.getLogger()
|
||||
log.level = logging.DEBUG
|
||||
|
||||
|
||||
def log_handler(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
log.addHandler(stream_handler)
|
||||
result = func(*args, **kwargs)
|
||||
log.removeHandler(stream_handler)
|
||||
return result
|
||||
return wrapper
|
||||
|
||||
|
||||
class TestComponentConfig(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.all_possible_components = {'comp1', 'comp2', 'comp3', 'comp4'}
|
||||
ComponentConfig.ScopeKeys = {'build', 'revalidate', '_scope_1', '_scope_2', '_scope_3'}
|
||||
|
||||
@log_handler
|
||||
def validate(self, config_data: dict, changed_components: set, expected_result: dict):
|
||||
log.info(f"{self._testMethodName}:")
|
||||
config = ComponentConfig(config_data, {}, self.all_possible_components)
|
||||
result = config.get_affected_components(changed_components)
|
||||
self.assertEqual(expected_result, result)
|
||||
|
||||
def test_no_changed_components(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = set()
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': ComponentConfig.FullScope,
|
||||
'comp3': ComponentConfig.FullScope,
|
||||
'comp4': ComponentConfig.FullScope,
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_all_components_changed(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1', 'comp2', 'comp3', 'comp4'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': ComponentConfig.FullScope,
|
||||
'comp3': ComponentConfig.FullScope,
|
||||
'comp4': ComponentConfig.FullScope,
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_changed_component_not_defined(self):
|
||||
config_data = {
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': ComponentConfig.FullScope,
|
||||
'comp3': ComponentConfig.FullScope,
|
||||
'comp4': ComponentConfig.FullScope,
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_component_changed_no_scope_keys(self):
|
||||
config_data = {
|
||||
'comp1': {},
|
||||
'comp2': {},
|
||||
'comp3': {},
|
||||
'comp4': {},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': ComponentConfig.FullScope,
|
||||
'comp3': ComponentConfig.FullScope,
|
||||
'comp4': ComponentConfig.FullScope,
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_one_component_changed_dependents_empty(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_not_changed_dependent_component(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {'comp2'}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': {'build'}
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_changed_dependent_component(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {'comp2'}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1', 'comp2'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': ComponentConfig.FullScope
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_dependent_component_multiple_parents(self):
|
||||
config_data = {
|
||||
'comp1': {'_scope_1': {'comp2'}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, '_scope_2': {'comp2'}, '_scope_3': {'comp2'}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1', 'comp3'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': {'_scope_1', '_scope_2', '_scope_3'},
|
||||
'comp3': ComponentConfig.FullScope
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_dependent_component_empty_scopes(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {}, 'revalidate': {'comp2'}},
|
||||
'comp2': {},
|
||||
'comp3': {},
|
||||
'comp4': {},
|
||||
}
|
||||
changed_components = {'comp1', 'comp3'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': ComponentConfig.FullScope,
|
||||
'comp3': ComponentConfig.FullScope,
|
||||
'comp4': ComponentConfig.FullScope
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_changed_component_empty_dependencies(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {}, 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_multiple_dependents(self):
|
||||
config_data = {
|
||||
'comp1': {'build': {'comp2'}, 'revalidate': {'comp3'}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {'comp4'}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': {'build'},
|
||||
'comp3': ComponentConfig.FullScope,
|
||||
# We don't consider dependencies of dependencies affected, so comp4 is not expected here
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
def test_all_as_dependents(self):
|
||||
config_data = {
|
||||
'comp1': {'build': 'all', 'revalidate': {}},
|
||||
'comp2': {'build': {}, 'revalidate': {}},
|
||||
'comp3': {'build': {}, 'revalidate': {}},
|
||||
'comp4': {'build': {}, 'revalidate': {}},
|
||||
}
|
||||
changed_components = {'comp1'}
|
||||
expected_result = {
|
||||
'comp1': ComponentConfig.FullScope,
|
||||
'comp2': {'build'},
|
||||
'comp3': {'build'},
|
||||
'comp4': {'build'},
|
||||
}
|
||||
self.validate(config_data, changed_components, expected_result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
name: 'System Information'
|
||||
description: 'Information about the system'
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
# Install pre-requisites for Fedora
|
||||
if [[ -e /etc/fedora-release ]]; then
|
||||
yum update -y -q && yum install -y -q procps
|
||||
fi
|
||||
|
||||
echo "System: ${{ runner.os }}"
|
||||
echo "System Architecture: ${{ runner.arch }}"
|
||||
echo "CPU Info: "; lscpu
|
||||
echo "RAM Info: "; free -h --si
|
||||
echo "MEMORY Info: "; df -h
|
||||
|
||||
- if: runner.os == 'macOS'
|
||||
shell: bash
|
||||
run: |
|
||||
echo "System: ${{ runner.os }}"
|
||||
echo "System Architecture: ${{ runner.arch }}"
|
||||
echo "CPU and RAM Info: "; system_profiler SPHardwareDataType
|
||||
echo "MEMORY Info: "; df -h
|
||||
|
||||
- if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
echo "System: ${{ runner.os }}"
|
||||
echo "System Architecture: ${{ runner.arch }}"
|
||||
echo "CPU Info: "; Get-CimInstance –ClassName Win32_Processor | Select-Object -Property Name, NumberOfCores, NumberOfLogicalProcessors
|
||||
echo "RAM info: $(systeminfo | Select-String 'Total Physical Memory:')"
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
Core:
|
||||
revalidate: 'all'
|
||||
|
||||
transformations:
|
||||
revalidate: 'all'
|
||||
|
||||
LP_transformations:
|
||||
revalidate:
|
||||
- CPU
|
||||
- GPU
|
||||
- PyTorch_FE
|
||||
- TF_FE
|
||||
- TFL_FE
|
||||
- ONNX_FE
|
||||
- PDPD_FE
|
||||
|
||||
preprocessing:
|
||||
revalidate:
|
||||
- inference
|
||||
- C_API
|
||||
- Python_API
|
||||
|
||||
inference:
|
||||
revalidate: 'all'
|
||||
|
||||
CPU:
|
||||
revalidate:
|
||||
- C_API
|
||||
- Python_API
|
||||
- samples
|
||||
- ONNX_RT
|
||||
- PyTorch_FE
|
||||
- TF_FE
|
||||
- ONNX_FE
|
||||
build:
|
||||
- AUTO
|
||||
- HETERO
|
||||
- AUTO_BATCH
|
||||
- TEMPLATE
|
||||
- IR_FE
|
||||
|
||||
GPU:
|
||||
build:
|
||||
- AUTO
|
||||
- HETERO
|
||||
- AUTO_BATCH
|
||||
- TEMPLATE
|
||||
- IR_FE
|
||||
- PROXY
|
||||
|
||||
NPU:
|
||||
revalidate: []
|
||||
build: []
|
||||
|
||||
HETERO:
|
||||
revalidate:
|
||||
- Python_API
|
||||
build:
|
||||
- TEMPLATE
|
||||
- IR_FE
|
||||
|
||||
AUTO_BATCH:
|
||||
revalidate:
|
||||
- GPU
|
||||
- C_API
|
||||
- Python_API
|
||||
build:
|
||||
- TEMPLATE
|
||||
- IR_FE
|
||||
|
||||
TEMPLATE:
|
||||
revalidate:
|
||||
- CPU
|
||||
- GPU
|
||||
- HETERO
|
||||
- AUTO_BATCH
|
||||
- AUTO
|
||||
- C_API
|
||||
- Python_API
|
||||
- NVIDIA
|
||||
build:
|
||||
- IR_FE
|
||||
|
||||
AUTO:
|
||||
revalidate:
|
||||
- C_API
|
||||
- Python_API
|
||||
build:
|
||||
- TEMPLATE
|
||||
- IR_FE
|
||||
|
||||
PROXY:
|
||||
revalidate:
|
||||
- inference
|
||||
- GPU
|
||||
build: []
|
||||
|
||||
IR_FE:
|
||||
revalidate:
|
||||
- C_API
|
||||
- Python_API
|
||||
- samples
|
||||
build:
|
||||
- CPU
|
||||
|
||||
ONNX_FE:
|
||||
revalidate:
|
||||
- MO
|
||||
- ONNX_RT
|
||||
build:
|
||||
- CPU
|
||||
- Python_API
|
||||
|
||||
PDPD_FE:
|
||||
revalidate:
|
||||
- MO
|
||||
build:
|
||||
- CPU
|
||||
- Python_API
|
||||
|
||||
TF_FE:
|
||||
revalidate:
|
||||
- MO
|
||||
build:
|
||||
- CPU
|
||||
- Python_API
|
||||
- TOKENIZERS
|
||||
|
||||
TFL_FE:
|
||||
revalidate:
|
||||
- MO
|
||||
build:
|
||||
- CPU
|
||||
- Python_API
|
||||
|
||||
PyTorch_FE:
|
||||
revalidate:
|
||||
- MO
|
||||
build:
|
||||
- CPU
|
||||
- Python_API
|
||||
- TOKENIZERS
|
||||
|
||||
C_API:
|
||||
build:
|
||||
- CPU
|
||||
- GPU
|
||||
- HETERO
|
||||
- AUTO_BATCH
|
||||
- AUTO
|
||||
- IR_FE
|
||||
|
||||
Python_API:
|
||||
revalidate:
|
||||
- samples
|
||||
- MO
|
||||
- tools
|
||||
build:
|
||||
- CPU
|
||||
- HETERO
|
||||
- AUTO_BATCH
|
||||
- TEMPLATE
|
||||
- AUTO
|
||||
- IR_FE
|
||||
- ONNX_FE
|
||||
- PDPD_FE
|
||||
- TF_FE
|
||||
- TFL_FE
|
||||
- PyTorch_FE
|
||||
|
||||
JS_API:
|
||||
build:
|
||||
- CPU
|
||||
- IR_FE
|
||||
|
||||
samples:
|
||||
build:
|
||||
- CPU
|
||||
- AUTO_BATCH
|
||||
- AUTO
|
||||
- IR_FE
|
||||
- C_API
|
||||
- Python_API
|
||||
|
||||
IE_Tests:
|
||||
revalidate:
|
||||
- CPU
|
||||
- GPU
|
||||
- HETERO
|
||||
- AUTO_BATCH
|
||||
- TEMPLATE
|
||||
- AUTO
|
||||
- NVIDIA
|
||||
build:
|
||||
- IR_FE
|
||||
|
||||
MO:
|
||||
revalidate:
|
||||
- PyTorch_FE
|
||||
- TF_FE
|
||||
build:
|
||||
- Python_API
|
||||
|
||||
tools:
|
||||
build:
|
||||
- CPU
|
||||
- Python_API
|
||||
|
||||
docs:
|
||||
revalidate: []
|
||||
build: []
|
||||
|
||||
licensing:
|
||||
revalidate: []
|
||||
build: []
|
||||
|
||||
NVIDIA:
|
||||
revalidate: []
|
||||
build: []
|
||||
|
||||
ONNX_RT:
|
||||
revalidate: []
|
||||
build: []
|
||||
|
||||
TOKENIZERS:
|
||||
revalidate: []
|
||||
build: []
|
||||
|
|
@ -1,143 +1,13 @@
|
|||
# See help here: https://docs.github.com/en/free-pro-team@latest/github/administering-a-repository/enabling-and-disabling-version-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
#
|
||||
# Python product dependencies
|
||||
#
|
||||
|
||||
# Python API, Frontends
|
||||
- package-ecosystem: pip
|
||||
directory: "/src/bindings/python/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Poland"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "rkazants"
|
||||
- "ceciliapeng2011"
|
||||
- "meiyang-intel"
|
||||
- "mbencer"
|
||||
- "tomdol"
|
||||
- "jane-intel"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
# Tests
|
||||
- package-ecosystem: pip
|
||||
directory: "/tests"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Poland"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "rkazants"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
# Model Optimizer, openvino_dev and Benchmark tool
|
||||
- package-ecosystem: pip
|
||||
directory: "/tools"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Dubai"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "rkazants"
|
||||
- "andrei-kochin"
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "Wovchena"
|
||||
allow:
|
||||
- dependency-name: "*"
|
||||
dependency-type: "production"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
#
|
||||
# Python Samples
|
||||
#
|
||||
|
||||
- package-ecosystem: pip
|
||||
directory: "/samples/python/hello_reshape_ssd/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Dubai"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "Wovchena"
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "rkazants"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
- package-ecosystem: pip
|
||||
directory: "/samples/python/classification_sample_async/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Dubai"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "Wovchena"
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "rkazants"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
- package-ecosystem: pip
|
||||
directory: "/samples/python/benchmark/bert_benchmark/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Dubai"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "Wovchena"
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "rkazants"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
- package-ecosystem: pip
|
||||
directory: "/samples/python/hello_classification/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Dubai"
|
||||
open-pull-requests-limit: 3
|
||||
assignees:
|
||||
- "Wovchena"
|
||||
- "jiwaszki"
|
||||
- "p-wysocki"
|
||||
- "akuporos"
|
||||
- "rkazants"
|
||||
versioning-strategy: increase-if-necessary
|
||||
|
||||
#
|
||||
# Github actions - CI
|
||||
#
|
||||
|
||||
# Github actions
|
||||
- package-ecosystem: github-actions
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
time: "09:00"
|
||||
timezone: "Asia/Dubai"
|
||||
assignees:
|
||||
- "akashchi"
|
||||
- "mryzhov"
|
||||
- "ilya-lavrenov"
|
||||
open-pull-requests-limit: 3
|
||||
- package-ecosystem: pip
|
||||
directory: "/ngraph/python"
|
||||
schedule:
|
||||
interval: weekly
|
||||
day: monday
|
||||
time: "13:00"
|
||||
open-pull-requests-limit: 10
|
||||
reviewers:
|
||||
- postrational
|
||||
labels:
|
||||
- dependencies
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
fail-on-severity: 'low'
|
||||
allow-licenses:
|
||||
- 'BSD-2-Clause'
|
||||
- 'BSD-3-Clause'
|
||||
- 'BSD-2-Clause AND BSD-3-Clause'
|
||||
- 'MIT'
|
||||
- 'Apache-2.0'
|
||||
- 'ISC'
|
||||
- 'Apache-2.0 AND MIT'
|
||||
- 'BlueOak-1.0.0'
|
||||
- '0BSD'
|
||||
- 'Python-2.0'
|
||||
- 'LGPL-3.0'
|
||||
fail-on-scopes:
|
||||
- 'runtime'
|
||||
- 'development'
|
||||
- 'unknown'
|
||||
license-check: true
|
||||
vulnerability-check: true
|
||||
allow-dependencies-licenses:
|
||||
- 'pkg:pypi/PyGithub@2.2.0'
|
||||
- 'pkg:pypi/psycopg2-binary'
|
||||
|
|
@ -1 +0,0 @@
|
|||
pr-24878
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
# Pythons
|
||||
python3.8-dev \
|
||||
python3.8-venv \
|
||||
python3.8-distutils \
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
libhdf5-dev \
|
||||
# For Java API
|
||||
default-jdk \
|
||||
# Compiler
|
||||
gcc-10 \
|
||||
g++-10 \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set gcc-10 as a default compiler
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \
|
||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
|
||||
|
||||
# Install sscache
|
||||
ARG SCCACHE_VERSION="v0.7.5"
|
||||
ENV SCCACHE_HOME="/opt/sccache" \
|
||||
SCCACHE_PATH="/opt/sccache/sccache"
|
||||
|
||||
RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
|
||||
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-aarch64-unknown-linux-musl.tar.gz" && \
|
||||
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
|
||||
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
# Pythons
|
||||
python3.8-dev \
|
||||
python3.8-venv \
|
||||
python3.8-distutils \
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
# For Java API
|
||||
default-jdk \
|
||||
# Compiler \
|
||||
gcc-10 \
|
||||
g++-10 \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set gcc-10 as a default compiler
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \
|
||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
|
||||
|
||||
# Install sscache
|
||||
ARG SCCACHE_VERSION="v0.7.5"
|
||||
ENV SCCACHE_HOME="/opt/sccache" \
|
||||
SCCACHE_PATH="/opt/sccache/sccache"
|
||||
|
||||
RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
|
||||
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
|
||||
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
|
||||
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/nvidia/cuda:11.8.0-runtime-ubuntu20.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
wget \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
# Pythons
|
||||
python3.8-dev \
|
||||
python3.8-venv \
|
||||
python3.8-distutils \
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
# For Java API
|
||||
default-jdk \
|
||||
# Compiler \
|
||||
gcc-10 \
|
||||
g++-10 \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set gcc-10 as a default compiler
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \
|
||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
|
||||
|
||||
# Install sscache
|
||||
ARG SCCACHE_VERSION="v0.7.5"
|
||||
ENV SCCACHE_HOME="/opt/sccache" \
|
||||
SCCACHE_PATH="/opt/sccache/sccache"
|
||||
|
||||
RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
|
||||
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
|
||||
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
|
||||
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
|
||||
|
||||
# Install CUDA
|
||||
RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin && \
|
||||
mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
|
||||
apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub && \
|
||||
add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
|
||||
RUN apt update && apt install -y \
|
||||
libcudnn8=8.9.4.*-1+cuda11.8 \
|
||||
libcudnn8-dev=8.9.4.*-1+cuda11.8 \
|
||||
libcudnn8-samples=8.9.4.*-1+cuda11.8 \
|
||||
cuda-runtime-11-8 \
|
||||
cuda-11-8 \
|
||||
libcutensor1=1.6.1.5-1 \
|
||||
libcutensor-dev=1.6.1.5-1 \
|
||||
cuda-drivers=520.61.05-1 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
libtbb2 \
|
||||
# Pythons
|
||||
python3.8-dev \
|
||||
python3.8-venv \
|
||||
python3.8-distutils \
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
# For Java API
|
||||
default-jdk \
|
||||
# Compiler \
|
||||
gcc-10 \
|
||||
g++-10 \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set gcc-10 as a default compiler
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \
|
||||
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30
|
||||
|
||||
# Install sscache
|
||||
ARG SCCACHE_VERSION="v0.7.5"
|
||||
ENV SCCACHE_HOME="/opt/sccache" \
|
||||
SCCACHE_PATH="/opt/sccache/sccache"
|
||||
|
||||
RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
|
||||
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
|
||||
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
|
||||
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
libtbb2 \
|
||||
# Pythons
|
||||
python3.8-dev \
|
||||
python3.8-venv \
|
||||
python3.8-distutils \
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
# For Java API
|
||||
default-jdk \
|
||||
# Compiler \
|
||||
clang \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set clang as a default compiler
|
||||
RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && \
|
||||
update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100
|
||||
|
||||
# Install sscache
|
||||
ARG SCCACHE_VERSION="v0.7.5"
|
||||
ENV SCCACHE_HOME="/opt/sccache" \
|
||||
SCCACHE_PATH="/opt/sccache/sccache"
|
||||
|
||||
RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
|
||||
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \
|
||||
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
|
||||
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
# Python
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
libhdf5-dev \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install openvino dependencies
|
||||
ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh
|
||||
RUN chmod +x /install_openvino_dependencies.sh && \
|
||||
/install_openvino_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
# Python
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
libhdf5-dev \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default instead of Python 3.8
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04
|
||||
|
||||
USER root
|
||||
|
||||
# APT configuration
|
||||
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
|
||||
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf
|
||||
|
||||
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||
TZ="Europe/London"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository --yes --no-update ppa:git-core/ppa && \
|
||||
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install \
|
||||
curl \
|
||||
git \
|
||||
ca-certificates \
|
||||
gpg-agent \
|
||||
tzdata \
|
||||
# Python
|
||||
python3.11-dev \
|
||||
python3.11-venv \
|
||||
python3.11-distutils \
|
||||
libhdf5-dev \
|
||||
&& \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install build dependencies
|
||||
ADD install_build_dependencies.sh /install_build_dependencies.sh
|
||||
RUN chmod +x /install_build_dependencies.sh && \
|
||||
/install_build_dependencies.sh && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Setup pip
|
||||
ENV PIP_VERSION="24.0"
|
||||
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
|
||||
python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
|
||||
rm -f get-pip.py
|
||||
|
||||
# Use Python 3.11 as default
|
||||
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
|
||||
RUN python3.11 -m venv venv
|
||||
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"
|
||||
|
||||
ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
|
||||
ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Check GitHub organization and invite members
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member,too-many-locals
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from argparse import ArgumentParser
|
||||
|
||||
sys.path.append(str(Path(__file__).resolve().parents[1]))
|
||||
from github_org_control.configs import Config
|
||||
from github_org_control.github_api import GithubOrgApi, get_dev_emails, print_users
|
||||
from github_org_control.ldap_api import LdapApi, print_user_info, InfoLevel
|
||||
|
||||
|
||||
def remove_members(gh_api, cfg_emails, org_emails, dev_emails, org_emails_no_in_ldap):
|
||||
"""Checks and remove members"""
|
||||
print(
|
||||
f"\n{'=' * 10} Check accounts below and remove from the GitHub organization or "
|
||||
f"configuration {'=' * 10}"
|
||||
)
|
||||
|
||||
cfg_emails_no_in_org = sorted(cfg_emails.difference(org_emails))
|
||||
print(
|
||||
f"\nCfg developer emails - absent in GitHub organization {len(cfg_emails_no_in_org)}:",
|
||||
"; ".join(cfg_emails_no_in_org),
|
||||
)
|
||||
|
||||
non_member_ignored_logins = set(Config().IGNORE_LOGINS).difference(
|
||||
set(gh_api.org_members_by_login.keys())
|
||||
)
|
||||
print(
|
||||
f"\nIgnored logins - absent in GitHub organization {len(non_member_ignored_logins)}:\n",
|
||||
"\n".join(non_member_ignored_logins),
|
||||
)
|
||||
|
||||
org_emails_no_in_dev = sorted(org_emails.difference(dev_emails))
|
||||
print(
|
||||
f"\nOrg member emails - absent in cfg and LDAP PDLs {len(org_emails_no_in_dev)}:",
|
||||
"; ".join(org_emails_no_in_dev),
|
||||
)
|
||||
|
||||
print(
|
||||
f"\nOrg member emails - absent in LDAP at all {len(org_emails_no_in_ldap)}:",
|
||||
"; ".join(sorted(org_emails_no_in_ldap)),
|
||||
)
|
||||
|
||||
print("\nOrg members - no real name:")
|
||||
members_to_fix_name = sorted(gh_api.members_to_fix_name, key=lambda member: member.email)
|
||||
print_users(members_to_fix_name)
|
||||
print(
|
||||
"\nOrg member emails - no real name:",
|
||||
"; ".join([member.email.lower() for member in members_to_fix_name]),
|
||||
)
|
||||
|
||||
print("\nOrg members - no Intel emails:")
|
||||
print_users(gh_api.members_to_remove)
|
||||
|
||||
gh_api.remove_users(org_emails_no_in_ldap | gh_api.members_to_remove)
|
||||
|
||||
|
||||
def main():
|
||||
"""The main entry point function"""
|
||||
arg_parser = ArgumentParser()
|
||||
arg_parser.add_argument(
|
||||
"--cfg-file",
|
||||
metavar="PATH",
|
||||
default=Config.default_cfg_path,
|
||||
help=f"Path to json configuration file, e.g. {Config.default_cfg_path}",
|
||||
)
|
||||
arg_parser.add_argument("--teams", action="store_true", help="Check GitHub teams")
|
||||
arg_parser.add_argument("--no-ldap", action="store_true", help="Don't use LDAP info")
|
||||
args, unknown_args = arg_parser.parse_known_args()
|
||||
|
||||
Config(args.cfg_file, unknown_args)
|
||||
gh_api = GithubOrgApi()
|
||||
|
||||
if args.teams:
|
||||
gh_api.get_org_teams()
|
||||
return
|
||||
|
||||
cfg_emails = get_dev_emails()
|
||||
print(f"\nCfg developer emails {len(cfg_emails)}:", "; ".join(sorted(cfg_emails)))
|
||||
|
||||
dev_emails = set()
|
||||
dev_emails.update(cfg_emails)
|
||||
|
||||
if not args.no_ldap:
|
||||
ldap_api = LdapApi()
|
||||
ldap_emails = ldap_api.get_user_emails()
|
||||
dev_emails.update(ldap_emails)
|
||||
print(f"\nLDAP developer emails {len(ldap_emails)}:", "; ".join(sorted(ldap_emails)))
|
||||
|
||||
cfg_emails_no_in_ldap = ldap_api.get_absent_emails(cfg_emails)
|
||||
print(
|
||||
f"\nCfg developer emails - absent in LDAP at all {len(cfg_emails_no_in_ldap)}:",
|
||||
"; ".join(sorted(cfg_emails_no_in_ldap)),
|
||||
)
|
||||
|
||||
cfg_ldap_inters = cfg_emails.intersection(ldap_emails)
|
||||
print(
|
||||
f"\nCfg developer emails - present in LDAP developers {len(cfg_ldap_inters)}:",
|
||||
"; ".join(sorted(cfg_ldap_inters)),
|
||||
)
|
||||
|
||||
org_emails = gh_api.get_org_emails()
|
||||
print(f"\nOrg emails {len(org_emails)}:", "; ".join(sorted(org_emails)))
|
||||
|
||||
org_emails_no_in_ldap = set()
|
||||
if not args.no_ldap:
|
||||
org_ldap_diff = org_emails.difference(ldap_emails)
|
||||
print(
|
||||
f"\nOrg member emails - absent in LDAP developers {len(org_ldap_diff)}:",
|
||||
"; ".join(sorted(org_ldap_diff)),
|
||||
)
|
||||
|
||||
for email in org_ldap_diff:
|
||||
user_info = ldap_api.get_user_info_by_email(email)
|
||||
if user_info:
|
||||
print_user_info(user_info, InfoLevel.PDL)
|
||||
else:
|
||||
org_emails_no_in_ldap.add(email)
|
||||
|
||||
org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
|
||||
invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
|
||||
print(f"\nInvite emails {len(invite_emails)}:", "; ".join(sorted(invite_emails)))
|
||||
|
||||
valid_github_users = gh_api.get_valid_github_users(invite_emails)
|
||||
gh_api.invite_users(valid_github_users)
|
||||
|
||||
remove_members(gh_api, cfg_emails, org_emails, dev_emails, org_emails_no_in_ldap)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,261 +0,0 @@
|
|||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Check GitHub PRs and set labels by type and categories, e.g. 'ExternalPR', 'category: ci'
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member
|
||||
|
||||
import re
|
||||
import sys
|
||||
import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from argparse import ArgumentParser
|
||||
|
||||
sys.path.append(str(Path(__file__).resolve().parents[1]))
|
||||
from github_org_control import github_api
|
||||
from github_org_control.configs import Config
|
||||
|
||||
|
||||
class PrType(Enum):
|
||||
"""Constants for type of GitHub pull request by author membership"""
|
||||
|
||||
EXTERNAL = "ExternalPR"
|
||||
INTEL = "ExternalIntelPR"
|
||||
ORG = "OpenvinoPR"
|
||||
BAD = "BadPR"
|
||||
|
||||
|
||||
def get_pr_labels(pull):
|
||||
"""Gets PR labels as set"""
|
||||
pr_lables = set()
|
||||
for label in pull.labels:
|
||||
pr_lables.add(label.name)
|
||||
return pr_lables
|
||||
|
||||
|
||||
def set_pr_labels(pull, labels):
|
||||
"""Sets new PR labels (all previously set labels are removed)"""
|
||||
if not labels or Config().DRY_RUN:
|
||||
return
|
||||
print("Set PR labels:", labels)
|
||||
# set_labels() should accept list but fails with empty "AssertionError:"
|
||||
pull.set_labels(labels)
|
||||
|
||||
|
||||
def add_pr_labels(pull, labels):
|
||||
"""Adds PR labels"""
|
||||
if not labels or Config().DRY_RUN:
|
||||
return
|
||||
print("Add PR labels:", labels)
|
||||
for label in labels:
|
||||
pull.add_to_labels(label)
|
||||
|
||||
|
||||
def get_pr_type_by_labels(pull):
|
||||
"""Gets PR type using labels"""
|
||||
pr_lables = get_pr_labels(pull)
|
||||
pr_types = set(type.value for type in PrType)
|
||||
pr_types_labels = pr_lables & pr_types
|
||||
if not pr_types_labels:
|
||||
return None
|
||||
if len(pr_types_labels) > 1:
|
||||
print(f"Duplicated labels: {pr_types_labels}")
|
||||
return PrType.BAD
|
||||
return PrType(PrType(pr_types_labels.pop()))
|
||||
|
||||
|
||||
def get_label_by_team_name_re(team_name):
|
||||
"""Generates label by PR reviwer team name using regular expressions"""
|
||||
if "admins" in team_name:
|
||||
return "category: ci"
|
||||
re_compile_label = re.compile(rf"{Config().GITHUB_REPO}-(.+)-maintainers")
|
||||
re_label = re_compile_label.match(team_name)
|
||||
if re_label:
|
||||
return f"category: {re_label.group(1).strip()}"
|
||||
return None
|
||||
|
||||
|
||||
def get_label_by_team_name_map(team_name):
|
||||
"""Generates label by PR reviwer team name using config map"""
|
||||
return Config().TEAM_TO_LABEL.get(team_name)
|
||||
|
||||
|
||||
def get_category_labels(pull):
|
||||
"""Gets list of category labels by all PR reviwer teams"""
|
||||
labels = []
|
||||
pr_lables = get_pr_labels(pull)
|
||||
for reviewer_team in pull.get_review_requests()[1]:
|
||||
reviewer_label = get_label_by_team_name_map(reviewer_team.name)
|
||||
if reviewer_label and reviewer_label not in pr_lables:
|
||||
labels.append(reviewer_label)
|
||||
return labels
|
||||
|
||||
|
||||
def get_pr_info_str(pull):
|
||||
"""Gets info about PR using a few workarounds"""
|
||||
pr_title = pull.title.encode("ASCII", "ignore").decode()
|
||||
|
||||
# Workaround for PyGithub issue: https://github.com/PyGithub/PyGithub/issues/512
|
||||
pr_created_at = pull.created_at.replace(tzinfo=datetime.timezone.utc).astimezone()
|
||||
|
||||
return (
|
||||
f"PR: {pull.number} - {pr_title} - Created: {pr_created_at} - "
|
||||
f"Labels: {get_pr_labels(pull)} - Type: {get_pr_type_by_labels(pull)}"
|
||||
)
|
||||
|
||||
|
||||
def update_labels(gh_api, pull, non_org_intel_pr_users, non_org_pr_users):
|
||||
"""Checks and updates labels"""
|
||||
print("Check and update labels:")
|
||||
pr_type_by_labels = get_pr_type_by_labels(pull)
|
||||
add_labels = []
|
||||
|
||||
# Checks PR source type
|
||||
if gh_api.is_org_user(pull.user):
|
||||
print(" - Org user")
|
||||
elif github_api.is_intel_email(pull.user.email) or github_api.is_intel_company(
|
||||
pull.user.company
|
||||
):
|
||||
print(" - Non org user with Intel email or company")
|
||||
non_org_intel_pr_users.add(pull.user)
|
||||
if pr_type_by_labels is not PrType.INTEL:
|
||||
print(f'NO "{PrType.INTEL.value}" label: ', end="")
|
||||
github_api.print_users(pull.user)
|
||||
add_labels.append(PrType.INTEL.value)
|
||||
elif github_api.is_user_ignored(pull.user):
|
||||
print(" - IGNORED non org user with NO Intel email or company")
|
||||
else:
|
||||
print(" - Non org user with NO Intel email or company")
|
||||
non_org_pr_users.add(pull.user)
|
||||
if pr_type_by_labels is not PrType.EXTERNAL:
|
||||
print(f'NO "{PrType.EXTERNAL.value}" label: ', end="")
|
||||
github_api.print_users(pull.user)
|
||||
add_labels.append(PrType.EXTERNAL.value)
|
||||
|
||||
add_labels += get_category_labels(pull)
|
||||
add_pr_labels(pull, add_labels)
|
||||
|
||||
|
||||
def get_wrong_commits(pull):
|
||||
"""Returns commits with incorrect user and email"""
|
||||
pr_author_email = (pull.user.email or "").lower()
|
||||
print("GitHub PR author email:", pr_author_email)
|
||||
print("Check commits:")
|
||||
wrong_commits = set()
|
||||
for commit in pull.get_commits():
|
||||
# import pprint; pprint.pprint(commit.raw_data)
|
||||
print("Commit SHA:", commit.sha)
|
||||
# Use raw data because commit author can be non GitHub user
|
||||
commit_author_email = (commit.raw_data["commit"]["author"]["email"] or "").lower()
|
||||
commit_committer_email = (commit.raw_data["commit"]["committer"]["email"] or "").lower()
|
||||
print(" Commit author email:", commit_author_email)
|
||||
print(" Commit committer email:", commit_committer_email)
|
||||
if not github_api.is_valid_user(commit.author):
|
||||
print(
|
||||
" ERROR: User with the commit author email is absent in GitHub:",
|
||||
commit.raw_data["commit"]["author"]["name"],
|
||||
)
|
||||
wrong_commits.add(commit.sha)
|
||||
if not github_api.is_valid_user(commit.committer):
|
||||
print(
|
||||
" ERROR: User with the commit committer email is absent in GitHub:",
|
||||
commit.raw_data["commit"]["committer"]["name"],
|
||||
)
|
||||
wrong_commits.add(commit.sha)
|
||||
if not commit.raw_data["commit"]["verification"]["verified"]:
|
||||
print(
|
||||
" WARNING: The commit is not verified. Reason:",
|
||||
commit.raw_data["commit"]["verification"]["reason"],
|
||||
)
|
||||
if pr_author_email != commit_author_email or pr_author_email != commit_committer_email:
|
||||
print(" WARNING: Commit emails and GitHub PR author public email are differnt")
|
||||
return wrong_commits
|
||||
|
||||
|
||||
def main():
|
||||
"""The main entry point function"""
|
||||
arg_parser = ArgumentParser()
|
||||
arg_parser.add_argument(
|
||||
"--cfg-file",
|
||||
metavar="PATH",
|
||||
default=Config.default_cfg_path,
|
||||
help=f"Path to json configuration file, e.g. {Config.default_cfg_path}",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
"--pr", metavar="NUMBER", help="Get GitHub pull request with the number"
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
"--pr-state",
|
||||
default="open",
|
||||
choices=["open", "closed"],
|
||||
help="Set GitHub pull request state",
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
"--newer", metavar="MINUTES", help="Get newly created GitHub pull request only"
|
||||
)
|
||||
arg_parser.add_argument(
|
||||
"--check-commits",
|
||||
action="store_true",
|
||||
help="Check and compare git commit email with GitHub account email",
|
||||
)
|
||||
args, unknown_args = arg_parser.parse_known_args()
|
||||
|
||||
Config(args.cfg_file, unknown_args)
|
||||
gh_api = github_api.GithubOrgApi()
|
||||
|
||||
if args.pr:
|
||||
pulls = [gh_api.repo.get_pull(int(args.pr))]
|
||||
else:
|
||||
pulls = gh_api.repo.get_pulls(state=args.pr_state)
|
||||
print(f"\nPRs count ({args.pr_state}):", pulls.totalCount)
|
||||
|
||||
if args.newer:
|
||||
pr_created_after = (
|
||||
datetime.datetime.now() - datetime.timedelta(minutes=int(args.newer))
|
||||
).astimezone()
|
||||
print("Checking PRs created after:", pr_created_after)
|
||||
|
||||
non_org_intel_pr_users = set()
|
||||
non_org_pr_users = set()
|
||||
wrong_pulls = {}
|
||||
|
||||
for pull in pulls:
|
||||
pr_created_at = pull.created_at.replace(tzinfo=datetime.timezone.utc).astimezone()
|
||||
if args.newer and pr_created_at <= pr_created_after:
|
||||
print(f"\nIGNORE: {get_pr_info_str(pull)}")
|
||||
continue
|
||||
|
||||
print(f"\n{get_pr_info_str(pull)}")
|
||||
if args.check_commits:
|
||||
wrong_commits = get_wrong_commits(pull)
|
||||
if wrong_commits:
|
||||
wrong_pulls[pull.number] = wrong_commits
|
||||
else:
|
||||
update_labels(gh_api, pull, non_org_intel_pr_users, non_org_pr_users)
|
||||
|
||||
if wrong_pulls:
|
||||
for pull_number, wrong_commits in wrong_pulls.items():
|
||||
print(
|
||||
f"\nERROR: Remove or replace wrong commits in the PR {pull_number}:\n ",
|
||||
"\n ".join(wrong_commits),
|
||||
)
|
||||
print(
|
||||
"\nAbout commit signature verification:\n ",
|
||||
"https://docs.github.com/en/github/authenticating-to-github/"
|
||||
"managing-commit-signature-verification/about-commit-signature-verification",
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if non_org_intel_pr_users:
|
||||
print("\nNon org user with Intel email or company:")
|
||||
github_api.print_users(non_org_intel_pr_users)
|
||||
if non_org_pr_users:
|
||||
print("\nNon org user with NO Intel email or company:")
|
||||
github_api.print_users(non_org_pr_users)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
{
|
||||
"GITHUB_TOKEN": "<Put token here or set as arg or as env variable>",
|
||||
"GITHUB_ORGANIZATION": "openvinotoolkit",
|
||||
"GITHUB_REPO": "openvino",
|
||||
"IGNORE_LOGINS": [
|
||||
"openvino-ci",
|
||||
"openvino-pushbot",
|
||||
"workbench-ci-bot",
|
||||
"sysicvvpux",
|
||||
"ote-ci-bot"
|
||||
],
|
||||
"MAX_MEMBERS_TO_REMOVE": 15,
|
||||
"EMAILS_FILE_PATH": "dev_emails-test.txt",
|
||||
"PROXIES": {
|
||||
"HTTP_PROXY": null,
|
||||
"HTTPS_PROXY": null,
|
||||
"NO_PROXY": "localhost,127.0.0.1,.intel.com"
|
||||
},
|
||||
"DRY_RUN": false,
|
||||
"TEAM_TO_LABEL": {
|
||||
"openvino-ci-maintainers": "category: CI",
|
||||
"openvino-maintainers": "category: inference",
|
||||
"openvino-docs-maintainers": "category: docs",
|
||||
"openvino-ie-maintainers": "category: inference",
|
||||
"openvino-ie-cpu-maintainers": "category: CPU",
|
||||
"openvino-ie-gpu-maintainers": "category: GPU",
|
||||
"openvino-ie-lpt-maintainers": "category: LP transformations",
|
||||
"openvino-ie-transformations-maintainers": "category: transformations",
|
||||
"openvino-ie-auto-multi-maintainers": "category: AUTO",
|
||||
"openvino-auto-batch-maintainers": "category: AUTO BATCH",
|
||||
"openvino-hetero-maintainers": "category: HETERO",
|
||||
"openvino-ie-python-api-maintainers": "category: Python API",
|
||||
"openvino-ie-template-maintainers": "category: TEMPLATE",
|
||||
"openvino-ir-frontend-maintainers": "category: IR FE",
|
||||
"openvino-ie-paddle-maintainers": "category: PDPD FE",
|
||||
"openvino-tf-frontend-maintainers": "category: TF FE",
|
||||
"openvino-onnx-frontend-maintainers": "category: ONNX FE",
|
||||
"openvino-ie-tests-maintainers": "category: IE Tests",
|
||||
"openvino-mo-maintainers": "category: MO",
|
||||
"openvino-ngraph-maintainers": "category: Core",
|
||||
"openvino-scripts-maintainers": "category: build",
|
||||
"openvino-tests-maintainers": "category: IE Tests",
|
||||
"openvino-tools-maintainers": "category: tools",
|
||||
"openvino-configuration-mgmt": "category: dependency_changes",
|
||||
"openvino-samples-maintainers": "category: samples",
|
||||
"openvino-c-api-maintainers": "category: C API"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,120 +0,0 @@
|
|||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Configurations management
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,broad-except
|
||||
|
||||
import os
|
||||
import sys
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
if sys.version_info[:2] < (3, 8):
|
||||
raise Exception("Python version must be >= 3.8")
|
||||
|
||||
|
||||
class ConfigException(Exception):
|
||||
"""Base configuration exception"""
|
||||
|
||||
|
||||
class Config:
|
||||
"""Configuration wrapper"""
|
||||
|
||||
_instance = None
|
||||
_properties = None
|
||||
|
||||
default_cfg_path = Path(__file__).resolve().parent / "config.json"
|
||||
|
||||
def __new__(cls, *_args, **_kwargs):
|
||||
if not Config._instance:
|
||||
Config._instance = super(Config, cls).__new__(cls)
|
||||
return Config._instance
|
||||
|
||||
def __init__(self, file_path=None, cli_args=None):
|
||||
"""
|
||||
:param file_path: Path to json configuration file
|
||||
:type file_path: String
|
||||
|
||||
:param args: List of argparse arguments with patterns: 'name=value' or 'name'
|
||||
:type args: list
|
||||
"""
|
||||
if Config._properties:
|
||||
return
|
||||
|
||||
self._file_path = file_path or Config.default_cfg_path
|
||||
self._cli_args = cli_args or []
|
||||
|
||||
self._json_cfg = {}
|
||||
self._args = {}
|
||||
|
||||
self._load_cfg()
|
||||
self._parse_cli_args()
|
||||
|
||||
Config._properties = {}
|
||||
for name, value in self._json_cfg.items():
|
||||
if hasattr(self, name):
|
||||
raise ConfigException(f"Duplicating prosperity: {name}")
|
||||
property_value = self._args.get(name) or os.getenv(name)
|
||||
if property_value:
|
||||
# Try to set prosperity_value as Python literal structures, e.g. DRY_RUN=False
|
||||
try:
|
||||
property_value = ast.literal_eval(property_value)
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(property_value, type(value)):
|
||||
raise ConfigException(f"Python type of {name} parameter must be {type(value)}")
|
||||
else:
|
||||
property_value = value
|
||||
Config._properties[name] = property_value
|
||||
|
||||
self.set_proxy()
|
||||
|
||||
def __getattr__(self, attr_name):
|
||||
if attr_name in self._properties:
|
||||
return self._properties.get(attr_name)
|
||||
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attr_name}'")
|
||||
|
||||
def _load_cfg(self):
|
||||
"""Load the json configuration file"""
|
||||
try:
|
||||
with open(self._file_path, encoding="utf-8") as conf:
|
||||
self._json_cfg = json.load(conf)
|
||||
except Exception as exc:
|
||||
raise ConfigException("Failed to load configuration from:", self._file_path) from exc
|
||||
|
||||
def _parse_cli_args(self):
|
||||
"""Parse argparse arguments with patterns: 'name=value' or 'name'"""
|
||||
for cli_arg in self._cli_args:
|
||||
arg = cli_arg.split("=")
|
||||
if arg[0] not in self._json_cfg:
|
||||
raise ConfigException(f"Unsupported argument: {arg}")
|
||||
self._args[arg[0]] = True if len(arg) == 1 else "=".join(arg[1:])
|
||||
|
||||
@property
|
||||
def properties(self):
|
||||
"""Get all properties as Dict"""
|
||||
return self._properties
|
||||
|
||||
def set_proxy(self):
|
||||
"""Set proxies"""
|
||||
for proxy_name, url in self._properties["PROXIES"].items():
|
||||
if url is not None:
|
||||
print(f"Set proxy: {proxy_name}={url}")
|
||||
os.environ[proxy_name] = url
|
||||
|
||||
|
||||
def _test():
|
||||
"""Test and debug"""
|
||||
print("Config.default_cfg_path:", Config.default_cfg_path)
|
||||
cfg = Config(cli_args=["DRY_RUN", 'PROXIES={"NO_PROXY": "localhost"}'])
|
||||
print("Config.properties:", cfg.properties)
|
||||
print("cfg.PROXIES:", cfg.PROXIES)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test()
|
||||
|
|
@ -1,384 +0,0 @@
|
|||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
GitHub API for controlling organization
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member
|
||||
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import typing
|
||||
from pathlib import Path
|
||||
|
||||
from github import Github, GithubException, RateLimitExceededException, IncompletableObject
|
||||
from github.PaginatedList import PaginatedList
|
||||
|
||||
sys.path.append(str(Path(__file__).resolve().parents[1]))
|
||||
from github_org_control.configs import Config
|
||||
|
||||
|
||||
class GithubApiException(Exception):
|
||||
"""Base GitHub API exception"""
|
||||
|
||||
|
||||
def is_valid_user(user):
|
||||
"""Checks that user is valid github.Github object"""
|
||||
try:
|
||||
return user and user.login
|
||||
except IncompletableObject:
|
||||
return False
|
||||
|
||||
|
||||
def is_user_ignored(user):
|
||||
"""Checks that user should be ignored"""
|
||||
if is_valid_user(user) and user.login.lower() not in Config().IGNORE_LOGINS:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_name(name):
|
||||
"""Checks that GitHub user's name is valid"""
|
||||
return name and len(name) >= 3 and " " in name
|
||||
|
||||
|
||||
def is_intel_email(email):
|
||||
"""Checks that email is valid Intel email"""
|
||||
return email and len(email) > 10 and " " not in email and email.lower().endswith("@intel.com")
|
||||
|
||||
|
||||
def is_intel_company(company):
|
||||
"""Checks that company contains intel"""
|
||||
return company and "intel" in company.lower()
|
||||
|
||||
|
||||
def is_valid_intel_user(user):
|
||||
"""Checks that user is valid GitHub and Intel user"""
|
||||
try:
|
||||
return is_valid_user(user) and is_valid_name(user.name) and is_intel_email(user.email)
|
||||
except IncompletableObject:
|
||||
return False
|
||||
|
||||
|
||||
def print_users(users):
|
||||
"""Print list of users in different formats: list, set, PaginatedList"""
|
||||
if isinstance(users, (list, set, PaginatedList)):
|
||||
users_count = users.totalCount if isinstance(users, PaginatedList) else len(users)
|
||||
print(f"GitHub users {users_count} (login - name - company - email - valid):")
|
||||
else:
|
||||
users = [users]
|
||||
for user in users:
|
||||
if not is_valid_user(user):
|
||||
print("WRONG GitHub user: ???")
|
||||
continue
|
||||
|
||||
try:
|
||||
name = user.name
|
||||
except IncompletableObject:
|
||||
name = "???"
|
||||
|
||||
try:
|
||||
company = user.company
|
||||
except IncompletableObject:
|
||||
company = "???"
|
||||
|
||||
try:
|
||||
email = user.email
|
||||
except IncompletableObject:
|
||||
email = "???"
|
||||
|
||||
valid_check = "OK" if is_valid_intel_user(user) else "FIX"
|
||||
if not is_intel_email(email):
|
||||
valid_check += " email"
|
||||
if not is_valid_name(name):
|
||||
valid_check += " name"
|
||||
print(f'{user.login} - "{name}" - "{company}" - {email} - {valid_check}')
|
||||
|
||||
|
||||
def get_dev_emails():
|
||||
"""
|
||||
Read a file with developer emails. Supported email formats
|
||||
first_name.last_name@intel.com
|
||||
Import from Outlook: Last_name, First_name <first_name.last_name@intel.com>
|
||||
"""
|
||||
re_email = re.compile(r".+<(.+)>")
|
||||
emails = set()
|
||||
cfg = Config()
|
||||
with open(cfg.properties["EMAILS_FILE_PATH"]) as file_obj:
|
||||
for line in file_obj:
|
||||
line = line.strip().lower()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
re_outlook_email = re_email.match(line)
|
||||
if re_outlook_email:
|
||||
line = re_outlook_email.group(1).strip()
|
||||
if not is_intel_email(line):
|
||||
print(f'Wrong email in {cfg.properties["EMAILS_FILE_PATH"]}: {line}')
|
||||
continue
|
||||
emails.add(line)
|
||||
return emails
|
||||
|
||||
|
||||
class GithubOrgApi:
|
||||
"""Common API for GitHub organization"""
|
||||
|
||||
def __init__(self):
|
||||
self._cfg = Config()
|
||||
self.github = Github(self._cfg.GITHUB_TOKEN)
|
||||
self.github_org = self.github.get_organization(self._cfg.GITHUB_ORGANIZATION)
|
||||
self.repo = self.github.get_repo(f"{self._cfg.GITHUB_ORGANIZATION}/{self._cfg.GITHUB_REPO}")
|
||||
self.github_users_by_email = {}
|
||||
self.org_members_by_login = {}
|
||||
self.members_to_remove = set()
|
||||
self.members_to_fix_name = set()
|
||||
|
||||
def is_org_user(self, user):
|
||||
"""Checks that user is a member of GitHub organization"""
|
||||
if is_valid_user(user):
|
||||
# user.get_organization_membership(self.github_org) doesn't work with org members
|
||||
# permissions, GITHUB_TOKEN must be org owner now
|
||||
return self.github_org.has_in_members(user)
|
||||
return False
|
||||
|
||||
def get_org_emails(self):
|
||||
"""Gets and prints emails of all GitHub organization members"""
|
||||
org_members = self.github_org.get_members()
|
||||
org_emails = set()
|
||||
|
||||
print(f"\nOrg members {org_members.totalCount} (login - name - company - email - valid):")
|
||||
for org_member in org_members:
|
||||
self.org_members_by_login[org_member.login.lower()] = org_member
|
||||
print_users(org_member)
|
||||
if is_intel_email(org_member.email):
|
||||
email = org_member.email.lower()
|
||||
org_emails.add(email)
|
||||
self.github_users_by_email[email] = org_member
|
||||
if not is_valid_name(org_member.name):
|
||||
self.members_to_fix_name.add(org_member)
|
||||
else:
|
||||
self.members_to_remove.add(org_member)
|
||||
|
||||
print("\nOrg members - no Intel emails:")
|
||||
print_users(self.members_to_remove)
|
||||
|
||||
print("\nOrg members - no real name:")
|
||||
print_users(self.members_to_fix_name)
|
||||
print(
|
||||
"\nOrg member emails - no real name:",
|
||||
"; ".join([member.email.lower() for member in self.members_to_fix_name]),
|
||||
)
|
||||
|
||||
return org_emails
|
||||
|
||||
def get_org_invitation_emails(self):
|
||||
"""Gets GitHub organization teams prints info"""
|
||||
org_invitations = self.github_org.invitations()
|
||||
org_invitation_emails = set()
|
||||
|
||||
print(
|
||||
f"\nOrg invitations {org_invitations.totalCount} "
|
||||
"(login - name - company - email - valid):"
|
||||
)
|
||||
for org_invitation in org_invitations:
|
||||
print_users(org_invitation)
|
||||
if is_user_ignored(org_invitation):
|
||||
continue
|
||||
if is_intel_email(org_invitation.email):
|
||||
org_invitation_emails.add(org_invitation.email.lower())
|
||||
else:
|
||||
print("Strange org invitation:", org_invitation)
|
||||
|
||||
print(
|
||||
f"\nOrg invitation emails {len(org_invitation_emails)}:",
|
||||
"; ".join(org_invitation_emails),
|
||||
)
|
||||
return org_invitation_emails
|
||||
|
||||
def get_org_teams(self):
|
||||
"""Gets GitHub organization teams prints info"""
|
||||
teams = []
|
||||
org_teams = self.github_org.get_teams()
|
||||
print("\nOrg teams count:", org_teams.totalCount)
|
||||
for team in org_teams:
|
||||
teams.append(team.name)
|
||||
print(f"\nTeam: {team.name} - parent: {team.parent}")
|
||||
|
||||
repos = team.get_repos()
|
||||
print("Repos:")
|
||||
for repo in repos:
|
||||
print(f" {repo.name} -", team.get_repo_permission(repo))
|
||||
|
||||
team_maintainers = team.get_members(role="maintainer")
|
||||
team_maintainer_logins = set()
|
||||
for maintainer in team_maintainers:
|
||||
team_maintainer_logins.add(maintainer.login)
|
||||
team_members = team.get_members(role="member")
|
||||
team_member_logins = set()
|
||||
for member in team_members:
|
||||
team_member_logins.add(member.login)
|
||||
members = team.get_members(role="all")
|
||||
member_emails = []
|
||||
print("Members (role - login - name - company - email - valid):")
|
||||
for user in members:
|
||||
if user.login in team_maintainer_logins:
|
||||
print(" Maintainer - ", end="")
|
||||
elif user.login in team_member_logins:
|
||||
print(" Member - ", end="")
|
||||
else:
|
||||
# It is not possible to check child teams members
|
||||
print(" ??? - ", end="")
|
||||
print_users(user)
|
||||
if is_intel_email(user.email) and not is_user_ignored(user):
|
||||
member_emails.append(user.email.lower())
|
||||
print(f"Intel emails {len(member_emails)}:", "; ".join(member_emails))
|
||||
return teams
|
||||
|
||||
def get_github_user_by_email(self, email):
|
||||
"""Gets GitHub user by email"""
|
||||
if email in self.github_users_by_email:
|
||||
return self.github_users_by_email.get(email)
|
||||
|
||||
def search_users():
|
||||
paginated_users = self.github.search_users(f"{email} in:email")
|
||||
# Minimize the GitHub Rate Limit
|
||||
users = []
|
||||
for user in paginated_users:
|
||||
users.append(user)
|
||||
if len(users) == 1:
|
||||
return users[0]
|
||||
if len(users) == 0:
|
||||
return None
|
||||
raise GithubApiException(
|
||||
f"ERROR: Found {len(users)} GitHub accounts with the same email {email}"
|
||||
)
|
||||
|
||||
try:
|
||||
user = search_users()
|
||||
except RateLimitExceededException:
|
||||
print("WARNING: RateLimitExceededException")
|
||||
time.sleep(30)
|
||||
user = search_users()
|
||||
self.github_users_by_email[email] = user
|
||||
|
||||
return user
|
||||
|
||||
def get_valid_github_users(self, emails):
|
||||
"""Gets valid GitHub users by email and prints status"""
|
||||
valid_users = set()
|
||||
wrong_emails = set()
|
||||
no_account_emails = set()
|
||||
no_account_names = set()
|
||||
print(f"\nGitHub users from {len(emails)} invite emails (email - status):")
|
||||
for email in emails:
|
||||
if not is_intel_email(email):
|
||||
print(f"{email} - Non Intel email")
|
||||
wrong_emails.add(email)
|
||||
continue
|
||||
|
||||
# You can make up to 30 requests per minute; https://developer.github.com/v3/search/
|
||||
time.sleep(2)
|
||||
user = self.get_github_user_by_email(email)
|
||||
|
||||
if not user:
|
||||
print(f"{email} - No valid GitHub account")
|
||||
no_account_emails.add(email)
|
||||
continue
|
||||
|
||||
if user.email and user.email.lower() == email:
|
||||
if is_valid_name(user.name):
|
||||
print(f"{email} - OK")
|
||||
valid_users.add(user)
|
||||
else:
|
||||
print(f"{email} - No valid name in GitHub account: ", end="")
|
||||
print_users(user)
|
||||
no_account_names.add(email)
|
||||
else:
|
||||
print(f"{email} - Non public or wrong email in GitHub account: ", end="")
|
||||
print_users(user)
|
||||
no_account_emails.add(email)
|
||||
|
||||
print("\nValid users:")
|
||||
print_users(valid_users)
|
||||
|
||||
print(f"\nWrong emails {len(wrong_emails)}:", "; ".join(wrong_emails))
|
||||
|
||||
print(
|
||||
f"\nIntel emails - No valid GitHub account {len(no_account_emails)}:",
|
||||
"; ".join(no_account_emails),
|
||||
)
|
||||
|
||||
print(
|
||||
f"\nIntel emails - No valid name in GitHub account {len(no_account_names)}:",
|
||||
"; ".join(no_account_names),
|
||||
)
|
||||
return valid_users
|
||||
|
||||
def invite_users(self, users):
|
||||
"""Invites users to GitHub organization and prints status"""
|
||||
if not isinstance(users, typing.Iterable):
|
||||
users = [users]
|
||||
print(f"\nInvite {len(users)} users:")
|
||||
|
||||
for user in users:
|
||||
if isinstance(user, str):
|
||||
print(f"Email: {user}")
|
||||
self.github_org.invite_user(email=user)
|
||||
else:
|
||||
print(f'{user.login} - "{user.name}" - {user.email} - ', end="")
|
||||
try:
|
||||
if is_user_ignored(user):
|
||||
print("Ignored")
|
||||
continue
|
||||
if self._cfg.DRY_RUN:
|
||||
print("Dry run")
|
||||
continue
|
||||
self.github_org.invite_user(user=user)
|
||||
print("OK")
|
||||
except GithubException as exc:
|
||||
print(f'FAIL: {exc.data["errors"][0]["message"]}')
|
||||
|
||||
def remove_users(self, users):
|
||||
"""Removes users from GitHub organization"""
|
||||
if not isinstance(users, typing.Iterable):
|
||||
users = [users]
|
||||
print(f"\nRemove {len(users)} users:")
|
||||
|
||||
dry_run = self._cfg.DRY_RUN
|
||||
if not dry_run and len(users) > self._cfg.MAX_MEMBERS_TO_REMOVE:
|
||||
print(
|
||||
"WARNING: Review is required for removing members more than "
|
||||
f"{self._cfg.MAX_MEMBERS_TO_REMOVE}"
|
||||
)
|
||||
# TODO: Add notification
|
||||
dry_run = True
|
||||
|
||||
for user in users:
|
||||
member = self.get_github_user_by_email(user) if isinstance(user, str) else user
|
||||
print(f'{member.login} - "{member.name}" - {member.email} - ', end="")
|
||||
try:
|
||||
if is_user_ignored(member):
|
||||
print("Ignored")
|
||||
continue
|
||||
if dry_run:
|
||||
print("Dry run")
|
||||
continue
|
||||
self.github_org.remove_from_membership(member)
|
||||
print("OK")
|
||||
except GithubException as exc:
|
||||
print(f'FAIL: {exc.data["errors"][0]["message"]}')
|
||||
|
||||
|
||||
def _test():
|
||||
"""Test and debug"""
|
||||
Config(cli_args=["DRY_RUN=True"])
|
||||
dev_emails = get_dev_emails()
|
||||
print("dev_emails:", dev_emails)
|
||||
|
||||
gh_api = GithubOrgApi()
|
||||
gh_api.get_org_emails()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test()
|
||||
|
|
@ -1,247 +0,0 @@
|
|||
# Copyright (C) 2018-2021 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Gets info about users and groups via LDAP
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member
|
||||
|
||||
import sys
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
from ldap3 import Server, Connection, ALL, SUBTREE
|
||||
|
||||
sys.path.append(str(Path(__file__).resolve().parents[1]))
|
||||
from github_org_control.configs import Config
|
||||
|
||||
|
||||
class LdapApiException(Exception):
|
||||
"""Base LDAP API exception"""
|
||||
|
||||
|
||||
class InfoLevel(Enum):
|
||||
"""Constants for printing user info from LDAP"""
|
||||
|
||||
PDL = "PDL" # Public Distribution List (group of e-mail addresses)
|
||||
FULL = "Full"
|
||||
|
||||
|
||||
def print_user_info(info, info_level=None):
|
||||
"""Pretty-print of a user info data structure (dict). info_level is the InfoLevel Enum"""
|
||||
if not info or not info.get("mail"):
|
||||
raise LdapApiException("ERROR: No info or absent mail")
|
||||
|
||||
def get_membership():
|
||||
if info_level == InfoLevel.PDL:
|
||||
membership_info = " PDLs:"
|
||||
elif info_level == InfoLevel.FULL:
|
||||
membership_info = " memberOf :"
|
||||
else:
|
||||
return ""
|
||||
# Grouping groups by purpose
|
||||
if info_level == InfoLevel.PDL:
|
||||
sort_key = lambda i: i.split(",", 1)[0].lower()
|
||||
else:
|
||||
sort_key = lambda i: i.split(",", 1)[1] + i.split(",", 1)[0].lower()
|
||||
for item in sorted(info["memberOf"], key=sort_key):
|
||||
if info_level == InfoLevel.PDL and "OU=Delegated" not in item:
|
||||
continue
|
||||
membership_info += f"\n {item}"
|
||||
return membership_info
|
||||
|
||||
try:
|
||||
text_info = (
|
||||
f'\n{info["cn"]} <{info["mail"]}>; {info["sAMAccountName"]}; {info["employeeID"]}'
|
||||
f'\n Org group: {info["intelSuperGroupDescr"]} ({info["intelSuperGroupShortName"]}) /'
|
||||
f' {info["intelGroupDescr"]} ({info["intelGroupShortName"]}) /'
|
||||
f' {info["intelDivisionDescr"]} ({info["intelDivisionShortName"]}) /'
|
||||
f' {info["intelOrgUnitDescr"]}'
|
||||
f'\n Manager: {info.get("manager")}'
|
||||
f'\n Location: {info["intelRegionCode"]} / {info["co"]} / {info["intelSiteCode"]} /'
|
||||
f' {info["intelBldgCode"]} ({info.get("intelSiteName")}) /'
|
||||
f' {info["physicalDeliveryOfficeName"]}'
|
||||
f'\n Other: {info["employeeType"]} | {info["intelExportCountryGroup"]} |'
|
||||
f' {info["whenCreated"]} | {info["intelCostCenterDescr"]} | {info["jobDescription"]}'
|
||||
)
|
||||
except Exception as exc:
|
||||
raise LdapApiException(
|
||||
f'ERROR: Failed to get info about "{info["mail"]}". '
|
||||
f"Exception occurred:\n{repr(exc)}"
|
||||
) from exc
|
||||
print(text_info)
|
||||
|
||||
membership = get_membership()
|
||||
if info_level == InfoLevel.PDL and membership:
|
||||
print(membership)
|
||||
elif info_level == InfoLevel.FULL:
|
||||
for key in sorted(info):
|
||||
if isinstance(info[key], list):
|
||||
if key == "memberOf":
|
||||
print(membership)
|
||||
else:
|
||||
print(f" {key} :")
|
||||
for item in info[key]:
|
||||
print(" ", item)
|
||||
else:
|
||||
print(f" {key} : {info[key]}")
|
||||
|
||||
|
||||
class LdapApi:
|
||||
"""LDAP API for getting user info and emails"""
|
||||
|
||||
_binary_blobs = ["thumbnailPhoto", "msExchUMSpokenName", "msExchBlockedSendersHash"]
|
||||
_check_existing = [
|
||||
"intelExportCountryGroup",
|
||||
"physicalDeliveryOfficeName",
|
||||
"intelSuperGroupShortName",
|
||||
"intelGroupShortName",
|
||||
"intelDivisionShortName",
|
||||
]
|
||||
|
||||
null = "<null>"
|
||||
|
||||
def __init__(self):
|
||||
self._cfg = Config()
|
||||
self.server = Server(self._cfg.LDAP_SERVER, get_info=ALL)
|
||||
self.connection = Connection(
|
||||
self.server, user=self._cfg.LDAP_USER, password=self._cfg.LDAP_PASSWORD, auto_bind=True
|
||||
)
|
||||
self.connection.bind()
|
||||
|
||||
def get_user_emails(self, groups=None):
|
||||
"""Gets emails of LDAP groups and sub-groups"""
|
||||
print("\nGet emails from LDAP groups:")
|
||||
processed_ldap_members = {}
|
||||
|
||||
def process_group_members(member, parent_group):
|
||||
if member in processed_ldap_members:
|
||||
processed_ldap_members[member]["parent_groups"].append(parent_group)
|
||||
print(
|
||||
"\nWARNING: Ignore LDAP member to avoid duplication and recursive cycling "
|
||||
f"of PDLs: {member}\n "
|
||||
f'email: {processed_ldap_members[member].get("email")}\n parent_groups:'
|
||||
)
|
||||
for group in processed_ldap_members[member].get("parent_groups", []):
|
||||
print(7 * " ", group)
|
||||
|
||||
return
|
||||
processed_ldap_members[member] = {"email": None, "parent_groups": [parent_group]}
|
||||
|
||||
# AD moves terminated users to the boneyard OU in case the user returns,
|
||||
# so it can be reactivated with little effort.
|
||||
# After 30 days it is removed and the unix personality becomes unlinked.
|
||||
if "OU=Boneyard" in member:
|
||||
return
|
||||
self.connection.search(
|
||||
member, r"(objectClass=*)", SUBTREE, attributes=["cn", "member", "mail"]
|
||||
)
|
||||
|
||||
# print(self.connection.entries)
|
||||
if not self.connection.response:
|
||||
raise LdapApiException(f"ERROR: empty response. LDAP member: {member}")
|
||||
|
||||
# Check that the member is worker.
|
||||
# The response can contain several items, but the first item is valid only
|
||||
if "OU=Workers" in member:
|
||||
if self.connection.response[0]["attributes"]["mail"]:
|
||||
processed_ldap_members[member]["email"] = self.connection.response[0][
|
||||
"attributes"
|
||||
]["mail"].lower()
|
||||
return
|
||||
raise LdapApiException(
|
||||
f"ERROR: no mail. LDAP worker: {member}\n" f"{self.connection.entries}"
|
||||
)
|
||||
|
||||
if len(self.connection.response) > 1:
|
||||
raise LdapApiException(
|
||||
f"ERROR: multiple responses for {member}: "
|
||||
f"{len(self.connection.response)}\n"
|
||||
f"{self.connection.entries}"
|
||||
)
|
||||
|
||||
if self.connection.response[0]["attributes"]["member"]:
|
||||
for group_member in self.connection.response[0]["attributes"]["member"]:
|
||||
process_group_members(group_member, member)
|
||||
else:
|
||||
print(f"\nERROR: no members in LDAP group: {member}\n{self.connection.entries}")
|
||||
|
||||
for group in groups or self._cfg.LDAP_PDLs:
|
||||
print("\nProcess ROOT LDAP group:", group)
|
||||
process_group_members(group, "ROOT")
|
||||
return {
|
||||
member.get("email") for member in processed_ldap_members.values() if member.get("email")
|
||||
}
|
||||
|
||||
def _get_user_info(self, query):
|
||||
"""Gets user info from LDAP as dict matching key and values pairs from query"""
|
||||
query_filter = "".join(f"({key}={value})" for key, value in query.items())
|
||||
|
||||
for domain in self._cfg.LDAP_DOMAINS:
|
||||
search_base = f"OU=Workers,DC={domain},DC=corp,DC=intel,DC=com"
|
||||
self.connection.search(
|
||||
search_base,
|
||||
f"(&(objectcategory=person)(objectclass=user)(intelflags=1){query_filter})",
|
||||
SUBTREE,
|
||||
attributes=["*"],
|
||||
)
|
||||
|
||||
if self.connection.response:
|
||||
if len(self.connection.response) > 1:
|
||||
raise LdapApiException(
|
||||
f"ERROR: multiple responses for {query_filter}: "
|
||||
f"{len(self.connection.response)}\n"
|
||||
f"{self.connection.entries}"
|
||||
)
|
||||
info = self.connection.response[0]["attributes"]
|
||||
|
||||
# remove long binary blobs
|
||||
for blob in LdapApi._binary_blobs:
|
||||
info[blob] = b""
|
||||
for key in LdapApi._check_existing:
|
||||
if not info.get(key):
|
||||
info[key] = LdapApi.null
|
||||
return info
|
||||
return {}
|
||||
|
||||
def get_user_info_by_idsid(self, idsid):
|
||||
"""Gets user info from LDAP as dict using account name for searching"""
|
||||
return self._get_user_info({"sAMAccountName": idsid})
|
||||
|
||||
def get_user_info_by_name(self, name):
|
||||
"""Gets user info from LDAP as dict using common name for searching"""
|
||||
return self._get_user_info({"cn": name})
|
||||
|
||||
def get_user_info_by_email(self, email):
|
||||
"""Gets user info from LDAP as dict using emails for searching"""
|
||||
return self._get_user_info({"mail": email})
|
||||
|
||||
def get_absent_emails(self, emails):
|
||||
"""Checks users by email in LDAP and returns absent emails"""
|
||||
absent_emails = set()
|
||||
for email in emails:
|
||||
if not self.get_user_info_by_email(email):
|
||||
absent_emails.add(email)
|
||||
return absent_emails
|
||||
|
||||
|
||||
def _test():
|
||||
"""Test and debug"""
|
||||
ldap = LdapApi()
|
||||
|
||||
emails = ldap.get_user_emails()
|
||||
print(f'\nLDAP emails count: {len(emails)}\n{"; ".join(emails)}')
|
||||
|
||||
emails = ["foo@intel.com"]
|
||||
|
||||
for email in emails:
|
||||
info = ldap.get_user_info_by_email(email)
|
||||
if info:
|
||||
print_user_info(info, InfoLevel.PDL)
|
||||
else:
|
||||
print(f"\n{email} - not found")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_test()
|
||||
|
|
@ -1 +0,0 @@
|
|||
pylint==2.11.1
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
PyGithub==1.55
|
||||
ldap3==2.7
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
'category: AUTO BATCH':
|
||||
- 'src/plugins/auto_batch/**/*'
|
||||
|
||||
'category: AUTO':
|
||||
- 'src/plugins/auto/**/*'
|
||||
|
||||
'category: build':
|
||||
- 'cmake/**/*'
|
||||
- '**/CMakeLists.txt'
|
||||
- '**/*.cmake'
|
||||
|
||||
'category: C API':
|
||||
- 'src/bindings/c/**/*'
|
||||
|
||||
'category: CI':
|
||||
- '.github/**/*'
|
||||
- '.ci/**/*'
|
||||
- 'Jenkinsfile'
|
||||
|
||||
'github_actions':
|
||||
- '.github/workflows/*'
|
||||
|
||||
'category: Core':
|
||||
- 'src/core/**/*'
|
||||
- 'src/common/itt/**/*'
|
||||
- 'src/common/util/**/*'
|
||||
- 'src/frontends/common/**/*'
|
||||
- 'src/common/conditional_compilation/**/*'
|
||||
|
||||
'category: CPP API':
|
||||
- 'src/inference/include/**/*'
|
||||
- 'src/core/include/**/*'
|
||||
- 'src/frontends/common/include/**/*'
|
||||
- 'src/frontends/onnx/frontend/include/**/*'
|
||||
- 'src/frontends/tensorflow/include/**/*'
|
||||
- 'src/frontends/tensorflow_lite/include/**/*'
|
||||
- 'src/frontends/pytorch/include/**/*'
|
||||
- 'src/frontends/paddle/include/**/*'
|
||||
|
||||
'category: CPU':
|
||||
- 'src/plugins/intel_cpu/**/*'
|
||||
- 'src/common/snippets/**/*'
|
||||
- 'thirdparty/xbyak/**/*'
|
||||
|
||||
'category: dependency_changes':
|
||||
- '**/requirement*.txt'
|
||||
- '**/constraints*.txt'
|
||||
- 'scripts/**/*'
|
||||
- '.gitmodules'
|
||||
- '**/setup.py'
|
||||
- 'conan.lock'
|
||||
- 'conanfile.txt'
|
||||
- 'vcpkg.json'
|
||||
- any: ['thirdparty/**/*',
|
||||
'!thirdparty/**/CMakeLists.txt']
|
||||
|
||||
'category: docs':
|
||||
- '**/*.md'
|
||||
- any: ['docs/**/*',
|
||||
'!docs/snippets/**/*']
|
||||
|
||||
'category: docs_snippets':
|
||||
- 'docs/snippets/**/*'
|
||||
|
||||
'category: extensions':
|
||||
- 'src/core/include/openvino/core/extension.hpp'
|
||||
- 'src/frontends/common/include/openvino/frontend/extension.hpp'
|
||||
- 'src/frontends/common/include/openvino/frontend/extension/**/*'
|
||||
|
||||
'category: GPU':
|
||||
- 'src/plugins/intel_gpu/**/*'
|
||||
- 'thirdparty/ocl/**/*'
|
||||
|
||||
'category: NPU':
|
||||
- 'src/plugins/intel_npu/**/*'
|
||||
|
||||
'category: HETERO':
|
||||
- 'src/plugins/hetero/**/*'
|
||||
|
||||
'category: PROXY':
|
||||
- 'src/plugins/proxy/**/*'
|
||||
|
||||
'category: IE Tests':
|
||||
- 'thirdparty/gtest/**/*'
|
||||
- 'src/frontends/tests/frontend/shared/**/*'
|
||||
- 'src/tests/**/*'
|
||||
|
||||
'category: inference':
|
||||
- 'src/inference/**/*'
|
||||
- 'src/cmake/**/*'
|
||||
|
||||
'category: IR FE':
|
||||
- 'src/frontends/ir/**/*'
|
||||
|
||||
'category: LP transformations':
|
||||
- 'src/common/low_precision_transformations/**/*'
|
||||
|
||||
'category: MO':
|
||||
- 'tools/mo/**/*'
|
||||
- 'tools/ovc/**/*'
|
||||
- 'tests/layer_tests/mo_python_api_tests/**/*'
|
||||
- 'tests/layer_tests/ovc_python_api_tests/**/*'
|
||||
|
||||
'category: ONNX FE':
|
||||
- 'src/frontends/onnx/**/*'
|
||||
- 'thirdparty/onnx/**/*'
|
||||
- 'tests/layer_tests/onnx_tests/**/*'
|
||||
|
||||
'category: packaging':
|
||||
- 'cmake/**/packaging/**/*'
|
||||
- 'src/bindings/python/wheel/**/*'
|
||||
- any: ['src/bindings/js/node/CMakeLists.txt',
|
||||
'src/bindings/js/node/package.json',
|
||||
'src/bindings/js/node/package-lock.json']
|
||||
- 'tools/openvino_dev/**/*'
|
||||
|
||||
'category: PDPD FE':
|
||||
- 'src/frontends/paddle/**/*'
|
||||
- 'tests/layer_tests/py_frontend_tests/test_paddle_frontend.py'
|
||||
|
||||
'category: preprocessing':
|
||||
- 'src/common/preprocessing/**/*'
|
||||
|
||||
'category: Python API':
|
||||
- 'src/bindings/python/**/*'
|
||||
|
||||
'category: JS API':
|
||||
- 'src/bindings/js/**/*'
|
||||
|
||||
'category: samples':
|
||||
- 'samples/**/*'
|
||||
- 'thirdparty/zlib/**/*'
|
||||
- 'thirdparty/gflags/**/*'
|
||||
- 'thirdparty/json/**/*'
|
||||
- 'thirdparty/cnpy/**/*'
|
||||
- 'tests/samples_tests/smoke_tests/**/*'
|
||||
|
||||
'category: TEMPLATE':
|
||||
- 'src/plugins/template/**/*'
|
||||
|
||||
'category: TF FE':
|
||||
- 'src/frontends/tensorflow/**/*'
|
||||
- 'src/frontends/tensorflow_common/**/*'
|
||||
- 'tests/layer_tests/tensorflow_tests/**/*'
|
||||
- 'tests/layer_tests/tensorflow2_keras_tests/**/*'
|
||||
- 'tests/layer_tests/jax_tests/**/*'
|
||||
- any: ['tests/model_hub_tests/**',
|
||||
'!tests/model_hub_tests/pytorch/**/*']
|
||||
|
||||
'category: TFL FE':
|
||||
- 'src/frontends/tensorflow_lite/**/*'
|
||||
- 'src/frontends/tensorflow_common/**/*'
|
||||
- 'tests/layer_tests/tensorflow_lite_tests/**/*'
|
||||
|
||||
'category: PyTorch FE':
|
||||
- 'src/frontends/pytorch/**/*'
|
||||
- 'tests/layer_tests/pytorch_tests/**/*'
|
||||
- 'src/bindings/python/src/openvino/frontend/pytorch/**/*'
|
||||
- 'tests/layer_tests/py_frontend_tests/test_torch_decoder.py'
|
||||
- 'tests/layer_tests/py_frontend_tests/test_torch_frontend.py'
|
||||
- any: ['tests/model_hub_tests/**',
|
||||
'!tests/model_hub_tests/tensorflow/**/*']
|
||||
|
||||
'category: tools':
|
||||
- any: ['tools/**',
|
||||
'!tools/mo/**/*',
|
||||
'!tools/ovc/**/*']
|
||||
|
||||
'category: transformations':
|
||||
- 'src/common/transformations/**/*'
|
||||
- 'src/common/offline_transformations/**/*'
|
||||
- 'src/core/pattern/**/*'
|
||||
- 'src/core/pass/**/*'
|
||||
- 'src/core/include/openvino/pass/**/*'
|
||||
|
||||
'category: licensing':
|
||||
- 'licensing/**/*'
|
||||
- 'LICENSE'
|
||||
|
||||
'category: docker_env':
|
||||
- '.dockerignore'
|
||||
- 'install_build_dependencies.sh'
|
||||
- '**/install_openvino_dependencies.sh'
|
||||
- '.github/actions/handle_docker/**/*'
|
||||
|
||||
'category: dockerfiles':
|
||||
- '.github/dockerfiles/**/*'
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
# Copyright (C) 2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Check GitHub organization and invite members
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
import github_api
|
||||
from configs import Config
|
||||
|
||||
|
||||
def main():
|
||||
"""The main entry point function"""
|
||||
arg_parser = ArgumentParser()
|
||||
arg_parser.add_argument("--cfg-file", metavar="PATH", default=Config.default_cfg_path,
|
||||
help=f"Path to json configuration file, e.g. {Config.default_cfg_path}")
|
||||
arg_parser.add_argument("--teams", action="store_true", help="Check GitHub teams")
|
||||
args, unknown_args = arg_parser.parse_known_args()
|
||||
|
||||
Config(args.cfg_file, unknown_args)
|
||||
gh_api = github_api.GithubOrgApi()
|
||||
|
||||
if args.teams:
|
||||
gh_api.get_org_teams()
|
||||
else:
|
||||
dev_emails = github_api.get_dev_emails()
|
||||
print(f'\nDeveloper emails {len(dev_emails)}:', '; '.join(dev_emails))
|
||||
|
||||
org_emails = gh_api.get_org_emails()
|
||||
print(f'\nOrg emails {len(org_emails)}:', '; '.join(org_emails))
|
||||
|
||||
org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
|
||||
|
||||
invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
|
||||
print(f'\nInvite emails {len(invite_emails)}:', '; '.join(invite_emails))
|
||||
|
||||
no_in_dev_emails = org_emails.difference(dev_emails)
|
||||
print(f'\nOrg members - no in developers list {len(no_in_dev_emails)}:',
|
||||
'; '.join(no_in_dev_emails))
|
||||
|
||||
valid_github_users = gh_api.get_valid_github_users(invite_emails)
|
||||
|
||||
gh_api.invite_users(valid_github_users)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
# Copyright (C) 2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Check GitHub PRs and set labels by type and categories, e.g. 'ExternalPR', 'category: ci'
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member
|
||||
|
||||
import re
|
||||
import datetime
|
||||
from argparse import ArgumentParser
|
||||
from enum import Enum
|
||||
|
||||
import github_api
|
||||
from configs import Config
|
||||
|
||||
|
||||
class PrType(Enum):
|
||||
"""Constants for type of GitHub pull request by author membership"""
|
||||
EXTERNAL = 'ExternalPR'
|
||||
INTEL = 'ExternalIntelPR'
|
||||
ORG = 'OpenvinoPR'
|
||||
BAD = 'BadPR'
|
||||
|
||||
|
||||
def get_pr_labels(pull):
|
||||
"""Gets PR labels as set"""
|
||||
pr_lables = set()
|
||||
for label in pull.labels:
|
||||
pr_lables.add(label.name)
|
||||
return pr_lables
|
||||
|
||||
|
||||
def set_pr_labels(pull, labels):
|
||||
"""Sets PR labels"""
|
||||
if not labels or Config().DRY_RUN:
|
||||
return
|
||||
print(f'Set PR labels:', labels)
|
||||
pull.set_labels(labels)
|
||||
|
||||
|
||||
def get_pr_type_by_labels(pull):
|
||||
"""Gets PR type using labels"""
|
||||
pr_lables = get_pr_labels(pull)
|
||||
pr_types = set(type.value for type in PrType)
|
||||
pr_types_labels = pr_lables & pr_types
|
||||
if not pr_types_labels:
|
||||
return None
|
||||
if len(pr_types_labels) > 1:
|
||||
print(f'Duplicated labels: {pr_types_labels}')
|
||||
return PrType.BAD
|
||||
return PrType(PrType(pr_types_labels.pop()))
|
||||
|
||||
|
||||
def get_label_by_team_name_re(team_name):
|
||||
"""Generates label by PR reviwer team name using regular expressions"""
|
||||
if 'admins' in team_name:
|
||||
return 'category: ci'
|
||||
re_compile_label = re.compile(rf'{Config().GITHUB_REPO}-(.+)-maintainers')
|
||||
re_label = re_compile_label.match(team_name)
|
||||
if re_label:
|
||||
return f'category: {re_label.group(1).strip()}'
|
||||
return None
|
||||
|
||||
|
||||
def get_label_by_team_name_map(team_name):
|
||||
"""Generates label by PR reviwer team name using config map"""
|
||||
return Config().TEAM_TO_LABEL.get(team_name)
|
||||
|
||||
|
||||
def get_category_labels(pull):
|
||||
"""Gets list of category labels by all PR reviwer teams"""
|
||||
labels = []
|
||||
pr_lables = get_pr_labels(pull)
|
||||
for reviewer_team in pull.get_review_requests()[1]:
|
||||
reviewer_label = get_label_by_team_name_map(reviewer_team.name)
|
||||
if reviewer_label and reviewer_label not in pr_lables:
|
||||
labels.append(reviewer_label)
|
||||
return labels
|
||||
|
||||
|
||||
def main():
|
||||
"""The main entry point function"""
|
||||
arg_parser = ArgumentParser()
|
||||
arg_parser.add_argument("--cfg-file", metavar="PATH", default=Config.default_cfg_path,
|
||||
help=f"Path to json configuration file, e.g. {Config.default_cfg_path}")
|
||||
arg_parser.add_argument("--pr", metavar="NUMBER",
|
||||
help="Get GitHub pull request with the number")
|
||||
arg_parser.add_argument("--pr-state", default="open", choices=["open", "closed"],
|
||||
help="Set GitHub pull request state")
|
||||
arg_parser.add_argument("--newer", metavar="MINUTES",
|
||||
help="Get newly created GitHub pull request only")
|
||||
args, unknown_args = arg_parser.parse_known_args()
|
||||
|
||||
Config(args.cfg_file, unknown_args)
|
||||
gh_api = github_api.GithubOrgApi()
|
||||
|
||||
if args.pr:
|
||||
pulls = [gh_api.repo.get_pull(int(args.pr))]
|
||||
else:
|
||||
pulls = gh_api.repo.get_pulls(state=args.pr_state)
|
||||
print(f'\nPRs count ({args.pr_state}):', pulls.totalCount)
|
||||
|
||||
if args.newer:
|
||||
pr_created_after = datetime.datetime.now() - datetime.timedelta(minutes=int(args.newer))
|
||||
print('PRs created after:', pr_created_after)
|
||||
non_org_intel_pr_users = set()
|
||||
non_org_pr_users = set()
|
||||
for pull in pulls:
|
||||
if args.newer and pull.created_at <= pr_created_after:
|
||||
print(f'\nIGNORE: {pull} - Created: {pull.created_at}')
|
||||
continue
|
||||
pr_lables = get_pr_labels(pull)
|
||||
pr_type_by_labels = get_pr_type_by_labels(pull)
|
||||
set_labels = []
|
||||
print(f'\n{pull} - Created: {pull.created_at} - Labels: {pr_lables} -',
|
||||
f'Type: {pr_type_by_labels}', end='')
|
||||
|
||||
# Checks PR source type
|
||||
if gh_api.is_org_user(pull.user):
|
||||
print(' - Org user')
|
||||
elif github_api.is_intel_email(pull.user.email) or \
|
||||
github_api.is_intel_company(pull.user.company):
|
||||
print(' - Non org user with Intel email or company')
|
||||
non_org_intel_pr_users.add(pull.user)
|
||||
if pr_type_by_labels is not PrType.INTEL:
|
||||
print(f'NO "{PrType.INTEL.value}" label: ', end='')
|
||||
github_api.print_users(pull.user)
|
||||
set_labels.append(PrType.INTEL.value)
|
||||
else:
|
||||
print(f' - Non org user with NO Intel email or company')
|
||||
non_org_pr_users.add(pull.user)
|
||||
if pr_type_by_labels is not PrType.EXTERNAL:
|
||||
print(f'NO "{PrType.EXTERNAL.value}" label: ', end='')
|
||||
github_api.print_users(pull.user)
|
||||
set_labels.append(PrType.EXTERNAL.value)
|
||||
|
||||
set_labels += get_category_labels(pull)
|
||||
set_pr_labels(pull, set_labels)
|
||||
|
||||
print(f'\nNon org user with Intel email or company:')
|
||||
github_api.print_users(non_org_intel_pr_users)
|
||||
print(f'\nNon org user with NO Intel email or company:')
|
||||
github_api.print_users(non_org_pr_users)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
{
|
||||
"GITHUB_TOKEN": "<Put token here or set as arg or as env variable>",
|
||||
"GITHUB_ORGANIZATION": "openvinotoolkit",
|
||||
"GITHUB_REPO": "openvino",
|
||||
"IGNORE_LOGINS": [
|
||||
"openvino-ci",
|
||||
"openvino-pushbot",
|
||||
"lab-nerval",
|
||||
"lab-nerval-onnx-ci"
|
||||
],
|
||||
"EMAILS_FILE_PATH": "dev_emails-test.txt",
|
||||
"PROXIES": {
|
||||
"HTTP_PROXY": null,
|
||||
"HTTPS_PROXY": null,
|
||||
"NO_PROXY": "localhost,127.0.0.1,.intel.com"
|
||||
},
|
||||
"DRY_RUN": false,
|
||||
"TEAM_TO_LABEL": {
|
||||
"openvino-admins": "category: CI",
|
||||
"openvino-maintainers": "category: IE common",
|
||||
"openvino-docs-maintainers": "category: docs",
|
||||
"openvino-ie-maintainers": "category: IE common",
|
||||
"openvino-ie-cpu-maintainers": "category: CPU",
|
||||
"openvino-ie-gna-maintainers": "category: GNA",
|
||||
"openvino-ie-gpu-maintainers": "category: GPU",
|
||||
"openvino-ie-lpt-maintainers": "category: LP transformations",
|
||||
"openvino-ie-multi-maintainers": "category: MULTI",
|
||||
"openvino-ie-python-api-maintainers": "category: python api",
|
||||
"openvino-ie-tests-maintainers": "category: IE Tests",
|
||||
"openvino-ie-vpu-maintainers": "category: VPU",
|
||||
"openvino-mo-maintainers": "category: MO",
|
||||
"openvino-ngraph-maintainers": "category: nGraph",
|
||||
"openvino-tests-maintainers": "category: IE Tests",
|
||||
"openvino-tools-maintainers": "category: tools"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
# Copyright (C) 2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
Configurations management
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,broad-except
|
||||
|
||||
import os
|
||||
import sys
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
if sys.hexversion < 0x3060000:
|
||||
raise Exception('Python version must be >= 3.6')
|
||||
|
||||
|
||||
class ConfigException(Exception):
|
||||
"""Base configuration exception"""
|
||||
|
||||
|
||||
class Config:
|
||||
"""Configuration wrapper"""
|
||||
_instance = None
|
||||
properties = None
|
||||
default_cfg_path = Path(__file__).resolve().parent / 'config.json'
|
||||
|
||||
def __new__(cls, *_args, **_kwargs):
|
||||
if not Config._instance:
|
||||
Config._instance = super(Config, cls).__new__(cls)
|
||||
return Config._instance
|
||||
|
||||
def __init__(self, file_path=None, cli_args=None):
|
||||
"""
|
||||
:param file_path: Path to json configuration file
|
||||
:type file_path: String
|
||||
|
||||
:param args: List of argparse arguments with patterns: 'name=value' or 'name'
|
||||
:type args: list
|
||||
"""
|
||||
if Config.properties:
|
||||
return
|
||||
|
||||
self._file_path = file_path or Config.default_cfg_path
|
||||
self._cli_args = cli_args or []
|
||||
|
||||
self._json_cfg = {}
|
||||
self._args = {}
|
||||
|
||||
self._load_cfg()
|
||||
self._parse_cli_args()
|
||||
|
||||
Config.properties = {}
|
||||
for name, value in self._json_cfg.items():
|
||||
if hasattr(self, name):
|
||||
raise ConfigException(f'Duplicating prosperity: {name}')
|
||||
prosperity_value = self._args.get(name) or os.getenv(name)
|
||||
if prosperity_value:
|
||||
# Try to set prosperity_value as Python literal structures, e.g. DRY_RUN=False
|
||||
try:
|
||||
prosperity_value = ast.literal_eval(prosperity_value)
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(prosperity_value, type(value)):
|
||||
raise ConfigException(f'Python type of {name} parameter must be {type(value)}')
|
||||
else:
|
||||
prosperity_value = value
|
||||
setattr(self, name, prosperity_value)
|
||||
Config.properties[name] = prosperity_value
|
||||
|
||||
self.set_proxy()
|
||||
|
||||
def _load_cfg(self):
|
||||
"""Load the json configuration file"""
|
||||
try:
|
||||
with open(self._file_path) as conf:
|
||||
self._json_cfg = json.load(conf)
|
||||
except:
|
||||
print('Failed to load configuration from:', self._file_path)
|
||||
raise
|
||||
|
||||
def _parse_cli_args(self):
|
||||
"""Parse argparse arguments with patterns: 'name=value' or 'name'"""
|
||||
for cli_arg in self._cli_args:
|
||||
arg = cli_arg.split('=')
|
||||
if arg[0] not in self._json_cfg:
|
||||
raise ConfigException(f'Unsupported argument: {arg}')
|
||||
self._args[arg[0]] = True if len(arg) == 1 else '='.join(arg[1:])
|
||||
|
||||
def get_properties(self):
|
||||
"""Get all properties as Dict"""
|
||||
return self.properties
|
||||
|
||||
def set_proxy(self):
|
||||
"""Set proxies"""
|
||||
for proxy_name, url in self.properties['PROXIES'].items():
|
||||
if url is not None:
|
||||
print(f'Set proxy: {proxy_name}={url}')
|
||||
os.environ[proxy_name] = url
|
||||
|
||||
|
||||
def _test():
|
||||
"""Test and debug"""
|
||||
print('Config.default_cfg_path:', Config.default_cfg_path)
|
||||
cfg = Config(cli_args=['DRY_RUN=True'])
|
||||
print('Config.properties:', cfg.get_properties())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
_test()
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
# Copyright (C) 2020 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
GitHub API for controlling organization
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme,no-member
|
||||
|
||||
import re
|
||||
import time
|
||||
|
||||
from github import Github, GithubException, RateLimitExceededException, IncompletableObject
|
||||
from github import UnknownObjectException
|
||||
from github.PaginatedList import PaginatedList
|
||||
|
||||
from configs import Config
|
||||
|
||||
|
||||
def is_valid_user(user):
|
||||
"""Checks that user is valid github.Github object"""
|
||||
try:
|
||||
return user and user.login
|
||||
except IncompletableObject:
|
||||
return False
|
||||
|
||||
|
||||
def is_user_ignored(user):
|
||||
"""Checks that user should be ignored"""
|
||||
cfg = Config()
|
||||
if is_valid_user(user) and user.login.lower() not in cfg.properties['IGNORE_LOGINS']:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_valid_name(name):
|
||||
"""Checks that GitHub user's name is valid"""
|
||||
return name and len(name) >= 3 and ' ' in name
|
||||
|
||||
|
||||
def is_intel_email(email):
|
||||
"""Checks that email is valid Intel email"""
|
||||
return email and len(email) > 10 and ' ' not in email and email.lower().endswith('@intel.com')
|
||||
|
||||
|
||||
def is_intel_company(company):
|
||||
"""Checks that company contains intel"""
|
||||
return company and 'intel' in company.lower()
|
||||
|
||||
|
||||
def is_valid_intel_user(user):
|
||||
"""Checks that user is valid GitHub and Intel user"""
|
||||
return is_valid_user(user) and (is_valid_name(user.name) and is_intel_email(user.email) or
|
||||
is_user_ignored(user))
|
||||
|
||||
|
||||
def print_users(users):
|
||||
"""Print list of users in different formats: list, set, PaginatedList"""
|
||||
if isinstance(users, (list, set, PaginatedList)):
|
||||
users_count = users.totalCount if isinstance(users, PaginatedList) else len(users)
|
||||
print(f'\nGitHub users {users_count} (login - name - company - email - valid):')
|
||||
else:
|
||||
users = [users]
|
||||
for user in users:
|
||||
if not is_valid_user(user):
|
||||
print('WRONG GitHub user: ???')
|
||||
continue
|
||||
valid_check = 'OK' if is_valid_intel_user(user) else 'FIX'
|
||||
if not is_intel_email(user.email):
|
||||
valid_check += ' email'
|
||||
if not is_valid_name(user.name):
|
||||
valid_check += ' name'
|
||||
print(f'{user.login} - "{user.name}" - "{user.company}" - {user.email} - {valid_check}')
|
||||
|
||||
|
||||
def get_dev_emails():
|
||||
"""
|
||||
Read a file with developer emails. Supported email formats
|
||||
first_name.last_name@intel.com
|
||||
Import from Outlook: Last_name, First_name <first_name.last_name@intel.com>
|
||||
"""
|
||||
re_email = re.compile(r'.+<(.+)>')
|
||||
emails = set()
|
||||
cfg = Config()
|
||||
with open(cfg.properties['EMAILS_FILE_PATH']) as file_obj:
|
||||
for line in file_obj:
|
||||
line = line.strip().lower()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
re_outlook_email = re_email.match(line)
|
||||
if re_outlook_email:
|
||||
line = re_outlook_email.group(1).strip()
|
||||
if not is_intel_email(line):
|
||||
print(f'Wrong email in {cfg.properties["EMAILS_FILE_PATH"]}: {line}')
|
||||
continue
|
||||
emails.add(line)
|
||||
return emails
|
||||
|
||||
|
||||
class GithubOrgApi:
|
||||
"""Common API for GitHub organization"""
|
||||
|
||||
def __init__(self):
|
||||
self._cfg = Config()
|
||||
self.github = Github(self._cfg.GITHUB_TOKEN)
|
||||
self.github_org = self.github.get_organization(self._cfg.GITHUB_ORGANIZATION)
|
||||
self.repo = self.github.get_repo(f'{self._cfg.GITHUB_ORGANIZATION}/'
|
||||
f'{self._cfg.GITHUB_REPO}')
|
||||
|
||||
def is_org_user(self, user):
|
||||
"""Checks that user is a member of GitHub organization"""
|
||||
if is_valid_user(user):
|
||||
try:
|
||||
membership = user.get_organization_membership(self.github_org)
|
||||
# membership.role can be 'member' or 'admin'
|
||||
if membership.state == 'active' and membership.role:
|
||||
return True
|
||||
except UnknownObjectException:
|
||||
pass
|
||||
return False
|
||||
|
||||
def get_org_emails(self):
|
||||
"""Gets and prints all emails of GitHub organization members"""
|
||||
org_members = self.github_org.get_members()
|
||||
org_emails = set()
|
||||
org_members_fix = set()
|
||||
org_emails_fix_name = set()
|
||||
org_logins_fix_intel_email = set()
|
||||
|
||||
print(f'\nOrg members {org_members.totalCount} (login - name - company - email - valid):')
|
||||
for org_member in org_members:
|
||||
print_users(org_member)
|
||||
if is_user_ignored(org_member):
|
||||
continue
|
||||
if is_intel_email(org_member.email):
|
||||
org_emails.add(org_member.email.lower())
|
||||
if not is_valid_name(org_member.name):
|
||||
org_members_fix.add(org_member)
|
||||
org_emails_fix_name.add(org_member.email.lower())
|
||||
else:
|
||||
org_members_fix.add(org_member)
|
||||
org_logins_fix_intel_email.add(org_member.login.lower())
|
||||
|
||||
print_users(org_members_fix)
|
||||
print(f'\nOrg members - no Intel emails {len(org_logins_fix_intel_email)}:',
|
||||
'; '.join(org_logins_fix_intel_email))
|
||||
print(f'\nOrg members - no real name {len(org_emails_fix_name)}:',
|
||||
'; '.join(org_emails_fix_name))
|
||||
return org_emails
|
||||
|
||||
def get_org_invitation_emails(self):
|
||||
"""Gets GitHub organization teams prints info"""
|
||||
org_invitations = self.github_org.invitations()
|
||||
org_invitation_emails = set()
|
||||
|
||||
print(f'\nOrg invitations {org_invitations.totalCount} (login - name - email - valid):')
|
||||
for org_invitation in org_invitations:
|
||||
# TODO: investigate GithubException while access to user name and enable print_users()
|
||||
# github.GithubException.IncompletableObject: 400 "Returned object contains no URL"
|
||||
#print_users(org_invitation)
|
||||
print(f'{org_invitation.login} - ??? - {org_invitation.email} - ???')
|
||||
if is_user_ignored(org_invitation):
|
||||
continue
|
||||
if is_intel_email(org_invitation.email):
|
||||
org_invitation_emails.add(org_invitation.email.lower())
|
||||
else:
|
||||
print('Strange org invitation:', org_invitation)
|
||||
|
||||
print(f'\nOrg invitation emails {len(org_invitation_emails)}:',
|
||||
'; '.join(org_invitation_emails))
|
||||
return org_invitation_emails
|
||||
|
||||
def get_org_teams(self):
|
||||
"""Gets GitHub organization teams prints info"""
|
||||
teams = []
|
||||
org_teams = self.github_org.get_teams()
|
||||
print('\nOrg teams count:', org_teams.totalCount)
|
||||
for team in org_teams:
|
||||
teams.append(team.name)
|
||||
print(f'\nTeam: {team.name} - parent: {team.parent}')
|
||||
|
||||
repos = team.get_repos()
|
||||
print('Repos:')
|
||||
for repo in repos:
|
||||
print(f' {repo.name} -', team.get_repo_permission(repo))
|
||||
|
||||
team_maintainers = team.get_members(role='maintainer')
|
||||
team_maintainer_logins = set()
|
||||
for maintainer in team_maintainers:
|
||||
team_maintainer_logins.add(maintainer.login)
|
||||
team_members = team.get_members(role='member')
|
||||
team_member_logins = set()
|
||||
for member in team_members:
|
||||
team_member_logins.add(member.login)
|
||||
members = team.get_members(role='all')
|
||||
member_emails = []
|
||||
print('Members (role - login - name - company - email - valid):')
|
||||
for user in members:
|
||||
if user.login in team_maintainer_logins:
|
||||
print(' Maintainer - ', end='')
|
||||
elif user.login in team_member_logins:
|
||||
print(' Member - ', end='')
|
||||
else:
|
||||
# It is not possible to check child teams members
|
||||
print(' ??? - ', end='')
|
||||
print_users(user)
|
||||
if is_intel_email(user.email) and not is_user_ignored(user):
|
||||
member_emails.append(user.email.lower())
|
||||
print(f'Intel emails {len(member_emails)}:', '; '.join(member_emails))
|
||||
return teams
|
||||
|
||||
def get_valid_github_users(self, emails):
|
||||
"""Gets valid GitHub users by email and prints status"""
|
||||
valid_users = set()
|
||||
no_account_emails = set()
|
||||
print(f'\nGitHub users from {len(emails)} invite emails (email - status):')
|
||||
for email in emails:
|
||||
if not is_intel_email(email):
|
||||
print(f'{email} - Non Intel email')
|
||||
continue
|
||||
|
||||
# You can make up to 30 requests per minute; https://developer.github.com/v3/search/
|
||||
# Sleep 2.4 sec is about 25 requests per minute
|
||||
time.sleep(2.4)
|
||||
try:
|
||||
users = self.github.search_users(f'{email} in:email')
|
||||
except RateLimitExceededException:
|
||||
time.sleep(5)
|
||||
users = self.github.search_users(f'{email} in:email')
|
||||
|
||||
if users.totalCount == 0:
|
||||
print(f'{email} - No valid GitHub account')
|
||||
no_account_emails.add(email)
|
||||
continue
|
||||
if users.totalCount > 1:
|
||||
print(f'{email} - Found {users.totalCount} GitHub accounts')
|
||||
for user in users:
|
||||
if user.email and user.email.lower() == email:
|
||||
print(f'{email} - OK')
|
||||
valid_users.add(user)
|
||||
else:
|
||||
print(f'{email} - Non public or wrong email - login: {user.login} - '
|
||||
f'email: {user.email}')
|
||||
print('Valid users count:', len(valid_users))
|
||||
print_users(valid_users)
|
||||
print(f'\nIntel emails - No valid GitHub account {len(no_account_emails)}:',
|
||||
'; '.join(no_account_emails))
|
||||
return valid_users
|
||||
|
||||
def invite_users(self, users):
|
||||
"""Invites users and prints status"""
|
||||
if isinstance(users, (list, set)):
|
||||
print(f'\nInvite {len(users)} users:')
|
||||
else:
|
||||
users = [users]
|
||||
|
||||
for user in users:
|
||||
if isinstance(user, str):
|
||||
print(f'Email: {user}')
|
||||
self.github_org.invite_user(email=user)
|
||||
else:
|
||||
print(f'{user.login} - "{user.name}" - {user.email} - ', end='')
|
||||
try:
|
||||
if is_user_ignored(user):
|
||||
print('Ignored')
|
||||
continue
|
||||
if not self._cfg.DRY_RUN:
|
||||
self.github_org.invite_user(user=user)
|
||||
print('OK')
|
||||
else:
|
||||
print('Dry run')
|
||||
except GithubException as exc:
|
||||
print(f'FAIL: {exc.data["errors"][0]["message"]}')
|
||||
|
||||
|
||||
def _test():
|
||||
"""Test and debug"""
|
||||
Config(cli_args=['DRY_RUN=True'])
|
||||
dev_emails = get_dev_emails()
|
||||
print('dev_emails:', dev_emails)
|
||||
|
||||
gh_api = GithubOrgApi()
|
||||
gh_api.get_org_emails()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
_test()
|
||||
|
|
@ -0,0 +1 @@
|
|||
PyGithub==1.51
|
||||
|
|
@ -0,0 +1 @@
|
|||
pylint==2.3.0
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
### Details:
|
||||
- *item1*
|
||||
- *...*
|
||||
|
||||
### Tickets:
|
||||
- *ticket-id*
|
||||
|
|
@ -1,225 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from github import Github
|
||||
from psycopg2 import sql
|
||||
import os
|
||||
import logging
|
||||
import psycopg2
|
||||
import dateutil
|
||||
import argparse
|
||||
|
||||
def init_logger():
|
||||
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
logging.basicConfig(level=LOGLEVEL,
|
||||
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
|
||||
datefmt='%m-%d-%Y %H:%M:%S')
|
||||
|
||||
def make_parser():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-r', '--repository-name', type=str, required=True,
|
||||
help='Repository name in OWNER/REPOSITORY format')
|
||||
parser.add_argument('--run-id', type=str, required=True,
|
||||
help='Workflow Run ID')
|
||||
|
||||
return parser
|
||||
|
||||
def create_db_tables(conn, cur):
|
||||
cur.execute('''CREATE TABLE IF NOT EXISTS workflow_runs(
|
||||
id SERIAL PRIMARY KEY,
|
||||
run_id BIGINT,
|
||||
html_url TEXT,
|
||||
name VARCHAR(255),
|
||||
run_started_at TIMESTAMP,
|
||||
created_at TIMESTAMP,
|
||||
updated_at TIMESTAMP,
|
||||
triggering_actor_login VARCHAR(255),
|
||||
conclusion VARCHAR(25),
|
||||
event VARCHAR(50),
|
||||
run_attempt INT,
|
||||
repository_full_name VARCHAR(255),
|
||||
head_repository_full_name VARCHAR(255),
|
||||
head_branch VARCHAR(255),
|
||||
status VARCHAR(25),
|
||||
display_title TEXT,
|
||||
path TEXT,
|
||||
total_duration_seconds INT
|
||||
);
|
||||
''')
|
||||
cur.execute('''CREATE TABLE IF NOT EXISTS workflow_jobs(
|
||||
id SERIAL PRIMARY KEY,
|
||||
job_id BIGINT,
|
||||
parent_run_id BIGINT,
|
||||
html_url TEXT,
|
||||
name VARCHAR(255),
|
||||
created_at TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
queued_duration_seconds INT,
|
||||
duration_seconds INT,
|
||||
runner_name VARCHAR(255),
|
||||
status VARCHAR(25),
|
||||
conclusion VARCHAR(25),
|
||||
head_branch VARCHAR(255),
|
||||
run_attempt INT,
|
||||
workflow_name TEXT
|
||||
);
|
||||
''')
|
||||
cur.execute('''CREATE TABLE IF NOT EXISTS workflow_steps(
|
||||
id SERIAL PRIMARY KEY,
|
||||
parent_job_id BIGINT,
|
||||
name VARCHAR(255),
|
||||
conclusion VARCHAR(25),
|
||||
number INT,
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
duration_seconds INT
|
||||
);
|
||||
''')
|
||||
conn.commit()
|
||||
|
||||
def main():
|
||||
init_logger()
|
||||
parser = make_parser()
|
||||
args = parser.parse_args()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
github_token = os.environ.get('GITHUB_TOKEN')
|
||||
if not github_token:
|
||||
raise ValueError('GITHUB_TOKEN environment variable is not set!')
|
||||
|
||||
run_id = args.run_id
|
||||
repo_name = args.repository_name
|
||||
|
||||
|
||||
# this should be specified in runner's env
|
||||
db_username = os.environ.get('PGUSER')
|
||||
db_password = os.environ.get('PGPASSWORD')
|
||||
db_host = os.environ.get('PGHOST')
|
||||
db_database = os.environ.get('PGDATABASE')
|
||||
db_port = os.environ.get('PGPORT')
|
||||
conn = psycopg2.connect(host=db_host,
|
||||
port=db_port,
|
||||
user=db_username,
|
||||
password=db_password,
|
||||
database=db_database)
|
||||
|
||||
# Create tables
|
||||
cur = conn.cursor()
|
||||
create_db_tables(conn, cur)
|
||||
|
||||
# Get the data
|
||||
g = Github(github_token)
|
||||
repo = g.get_repo(repo_name)
|
||||
|
||||
run = repo.get_workflow_run(int(run_id))
|
||||
logger.info('Processing run ID %s - %s, URL: %s', run_id, run.name, run.html_url)
|
||||
if run.status != 'completed':
|
||||
logger.error('Run %s is not completed! Only completed runs should be in the database', run_id)
|
||||
raise SystemExit(1)
|
||||
|
||||
# We rely on the following assumptions:
|
||||
# - The workflow run is completed. When run.status != 'completed' we should not add it to the database
|
||||
# theoretically the second attempt can be triggerred right after the completion of the first one
|
||||
# or while the runner which executes this script is deploying
|
||||
#
|
||||
# - Job's queued duration equals "job.started_at - job.created_at" if started_at > created_at.
|
||||
# Otherwise the job should not be added to the database
|
||||
total_duration_seconds = round(run.timing().run_duration_ms / 1000)
|
||||
|
||||
workflow_data_query = sql.SQL('''INSERT INTO workflow_runs(
|
||||
run_id, html_url, name,
|
||||
run_started_at, created_at, updated_at, triggering_actor_login, conclusion,
|
||||
event, run_attempt, repository_full_name, head_repository_full_name,
|
||||
head_branch, status, display_title, path, total_duration_seconds)
|
||||
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
|
||||
''')
|
||||
|
||||
workflow_query_args = (run_id, run.html_url, run.name,
|
||||
run.run_started_at, run.created_at, run.updated_at,
|
||||
run.raw_data['triggering_actor']['login'],
|
||||
run.conclusion, run.event,
|
||||
run.run_attempt, run.raw_data['repository']['full_name'],
|
||||
run.raw_data['head_repository']['full_name'],
|
||||
run.head_branch, run.status,
|
||||
run.display_title, run.path, total_duration_seconds)
|
||||
|
||||
logger.debug('Workflow run query: %s', cur.mogrify(workflow_data_query, workflow_query_args))
|
||||
cur.execute(workflow_data_query, workflow_query_args)
|
||||
|
||||
for job in run.jobs():
|
||||
job_id = job.id
|
||||
logger.info('Processing job %s, URL: %s', job.name, job.html_url)
|
||||
queued_duration_seconds = 0
|
||||
duration_seconds = 0
|
||||
|
||||
job_created_at_date = dateutil.parser.parse(job.raw_data['created_at'])
|
||||
if job_created_at_date > job.started_at:
|
||||
logger.warning('Skipping job %s of run %s - most likely a stub \
|
||||
job created after workflow restart', job.name, run_id)
|
||||
continue
|
||||
|
||||
queued_duration_timedelta = job.started_at - job_created_at_date
|
||||
queued_duration_seconds = round(queued_duration_timedelta.total_seconds())
|
||||
|
||||
duration_timedelta = job.completed_at - job.started_at
|
||||
duration_seconds = round(duration_timedelta.total_seconds())
|
||||
|
||||
job_data_query = sql.SQL('''
|
||||
INSERT INTO workflow_jobs(
|
||||
job_id, parent_run_id, html_url, name,
|
||||
created_at, started_at, completed_at,
|
||||
queued_duration_seconds, duration_seconds,
|
||||
runner_name, status, conclusion, head_branch,
|
||||
run_attempt, workflow_name
|
||||
)
|
||||
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
|
||||
''')
|
||||
|
||||
job_query_args = (job_id, run_id, job.html_url, job.name,
|
||||
job.raw_data['created_at'], job.started_at, job.completed_at,
|
||||
queued_duration_seconds, duration_seconds,
|
||||
job.raw_data['runner_name'], job.status, job.conclusion,
|
||||
job.raw_data['head_branch'], job.raw_data['run_attempt'], job.raw_data['workflow_name'])
|
||||
|
||||
logger.debug('Job query: %s', cur.mogrify(job_data_query, job_query_args))
|
||||
cur.execute(job_data_query, job_query_args)
|
||||
conn.commit()
|
||||
for step in job.steps:
|
||||
logger.info('Processing step %s', step.name)
|
||||
|
||||
# It seems like a GitHub bug, but
|
||||
# sometimes there're steps "In progress"
|
||||
# despite the workflow being completed
|
||||
step_duration_seconds = 0
|
||||
|
||||
if step.status != 'completed':
|
||||
logger.warning('Step %s isn\'t completed. Setting duration to zero.', step.name)
|
||||
step_duration_seconds = 0
|
||||
else:
|
||||
step_duration_seconds_timedelta = step.completed_at - step.started_at
|
||||
step_duration_seconds = round(step_duration_seconds_timedelta.total_seconds())
|
||||
|
||||
step_data_query = sql.SQL('''
|
||||
INSERT INTO workflow_steps(
|
||||
parent_job_id, name, conclusion,
|
||||
number, started_at, completed_at,
|
||||
duration_seconds)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s);
|
||||
''')
|
||||
step_query_args = (
|
||||
job_id, step.name, step.conclusion,
|
||||
step.number, step.started_at, step.completed_at,
|
||||
step_duration_seconds)
|
||||
|
||||
logger.debug('Step query: %s', cur.mogrify(step_data_query, step_query_args))
|
||||
cur.execute(step_data_query, step_query_args)
|
||||
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
g.close()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
from enum import Enum
|
||||
|
||||
from github import Github, Auth
|
||||
|
||||
|
||||
class ExternalPRLabels(str, Enum):
|
||||
ExternalPR = 'ExternalPR'
|
||||
ExternalIntelPR = 'ExternalIntelPR'
|
||||
|
||||
|
||||
def get_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-r',
|
||||
'--repository-name',
|
||||
type=str,
|
||||
required=True,
|
||||
help='Repository name in the OWNER/REPOSITORY format',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--pr-number', type=int, required=True, help='PR number to label'
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def init_logger():
|
||||
logging.basicConfig(
|
||||
level=os.environ.get('LOGLEVEL', 'INFO').upper(),
|
||||
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
|
||||
datefmt='%m-%d-%Y %H:%M:%S',
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
init_logger()
|
||||
|
||||
LOGGER = logging.getLogger('labeller')
|
||||
|
||||
args = get_arguments()
|
||||
pr_number = args.pr_number
|
||||
repository_name = args.repository_name
|
||||
|
||||
github = Github(auth=Auth.Token(token=os.environ.get('GITHUB_TOKEN')))
|
||||
gh_repo = github.get_repo(full_name_or_id=repository_name)
|
||||
|
||||
pr = gh_repo.get_pull(number=pr_number)
|
||||
|
||||
LOGGER.info(f'CONTEXT: PR #{pr_number}. USER: {pr.user.login}. ALL PR LABELS: {list(pr.get_labels())}')
|
||||
|
||||
if not gh_repo.has_in_collaborators(pr.user.login):
|
||||
LOGGER.info(f'THE {pr.user.login} IS NOT A COLLABORATOR')
|
||||
|
||||
for label in pr.get_labels():
|
||||
if label.name in (ExternalPRLabels.ExternalPR, ExternalPRLabels.ExternalIntelPR):
|
||||
LOGGER.info(f'THE PR ALREADY HAS THE "{label.name}" LABEL')
|
||||
break
|
||||
else:
|
||||
is_intel_user = bool(pr.user.email and pr.user.email.endswith('@intel.com'))
|
||||
label_to_add: str = ExternalPRLabels.ExternalIntelPR.name if is_intel_user else ExternalPRLabels.ExternalPR.name
|
||||
|
||||
pr.add_to_labels(label_to_add)
|
||||
LOGGER.info(f'THE "{label_to_add}" LABEL WAS ADDED TO THE PR')
|
||||
else:
|
||||
LOGGER.info(
|
||||
f'THE {pr.user.login} IS A COLLABORATOR, NO NEED TO ADD THE "External" LABEL'
|
||||
)
|
||||
|
||||
github.close()
|
||||
|
|
@ -1 +0,0 @@
|
|||
python-dateutil==2.9.0.post0
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('-r', '--repository-name',
|
||||
type=str,
|
||||
required=True,
|
||||
help='Repository name in the OWNER/REPOSITORY format')
|
||||
parser.add_argument('--run-id',
|
||||
type=int,
|
||||
required=True,
|
||||
help='Workflow Run ID')
|
||||
parser.add_argument('--errors-to-look-for-file',
|
||||
type=Path,
|
||||
required=False,
|
||||
help='.json file with the errors to look for in logs',
|
||||
default=Path(__file__).resolve().parent.joinpath('errors_to_look_for.json'))
|
||||
parser.add_argument('--dry-run',
|
||||
required=False,
|
||||
action='store_true',
|
||||
help='Whether to run in dry mode and not actually retrigger the pipeline'
|
||||
' and only collect and analyze logs')
|
||||
return parser.parse_args()
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
# Copyright (C) 2024 Intel Corporation
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN')
|
||||
|
||||
|
||||
def init_logger():
|
||||
LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
|
||||
logging.basicConfig(level=LOGLEVEL,
|
||||
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
|
||||
datefmt='%m-%d-%Y %H:%M:%S')
|
||||
|
||||
|
||||
init_logger()
|
||||
|
||||
LOGGER = logging.getLogger('rerunner')
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue