From bcbb4d4c0774bab09f62fdf09897e9c6b30357bc Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Tue, 3 Mar 2026 12:37:45 -0500 Subject: [PATCH] fix: Added retries for docker pull. Removed unused docker-tag-push GH action (#6804) Signed-off-by: Dmitry Tokarev --- .github/actions/docker-tag-push/action.yml | 86 ------------------- .github/actions/docker-tag-push/retry_push.sh | 29 ------- .github/scripts/retry_docker.sh | 45 ++++++++++ .../build-test-distribute-flavor.yml | 10 ++- .../workflows/container-validation-dynamo.yml | 12 ++- 5 files changed, 60 insertions(+), 122 deletions(-) delete mode 100644 .github/actions/docker-tag-push/action.yml delete mode 100644 .github/actions/docker-tag-push/retry_push.sh create mode 100644 .github/scripts/retry_docker.sh diff --git a/.github/actions/docker-tag-push/action.yml b/.github/actions/docker-tag-push/action.yml deleted file mode 100644 index 39138a28c..000000000 --- a/.github/actions/docker-tag-push/action.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: 'Docker Tag and Push' -description: 'Tag and Push Docker Images' - -inputs: - local_image: - description: 'Local Image Name:Tag' - required: true - push_tags: - description: 'Target Name:Tag (newline-separated list for multiple tags)' - required: true - # There isn't a clean way to have an additional tag that is conditional - # Adding this to handle this use-case (we want multiple tags for main builds) - conditional_tag: - description: 'Optional tag for conditionals' - required: false - aws_push: - description: 'Push to AWS Boolean' - required: false - default: 'false' - azure_push: - description: 'Push to Azure Container Registry (ACR) Boolean' - required: false - default: 'false' - aws_account_id: - description: 'AWS Account ID' - required: false - aws_default_region: - description: 'AWS Default Region' - required: false - azure_acr_hostname: - description: 'Azure ACR hostname' - required: false - -outputs: - image_tags: - description: 'Image Tags' - value: ${{ inputs.push_tags }} - -runs: - using: "composite" - steps: - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: ECR Tag and Push - shell: bash - if: ${{ inputs.aws_push == 'true' }} - env: - LOCAL_IMAGE: ${{ inputs.local_image }} - PUSH_TAGS: ${{ inputs.push_tags }} - CONDITIONAL_TAG: ${{ inputs.conditional_tag }} - ECR_HOSTNAME: ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_default_region }}.amazonaws.com - run: | - set -euo pipefail - source "${{ github.action_path }}/retry_push.sh" - - if [[ -n "${CONDITIONAL_TAG}" ]]; then - docker tag "${LOCAL_IMAGE}" "${ECR_HOSTNAME}/${CONDITIONAL_TAG}" - retry_push "${ECR_HOSTNAME}/${CONDITIONAL_TAG}" - fi - while IFS= read -r TAG; do - if [ -z "$TAG" ]; then - continue - fi - echo "Tagging and pushing: ${ECR_HOSTNAME}/${TAG}" - docker tag "${LOCAL_IMAGE}" "${ECR_HOSTNAME}/${TAG}" - retry_push "${ECR_HOSTNAME}/${TAG}" - done <<< "$PUSH_TAGS" - - name: ACR Tag and Push - shell: bash - if: ${{ inputs.azure_push == 'true' }} - env: - LOCAL_IMAGE: ${{ inputs.local_image }} - PUSH_TAGS: ${{ inputs.push_tags }} - AZURE_ACR_HOSTNAME: ${{ inputs.azure_acr_hostname }} - run: | - set -euo pipefail - source "${{ github.action_path }}/retry_push.sh" - - while IFS= read -r TAG; do - if [ -z "$TAG" ]; then - continue - fi - echo "Tagging and pushing: ${AZURE_ACR_HOSTNAME}/${TAG}" - docker tag "${LOCAL_IMAGE}" "${AZURE_ACR_HOSTNAME}/${TAG}" - retry_push "${AZURE_ACR_HOSTNAME}/${TAG}" - done <<< "$PUSH_TAGS" diff --git a/.github/actions/docker-tag-push/retry_push.sh b/.github/actions/docker-tag-push/retry_push.sh deleted file mode 100644 index 58abe7b3d..000000000 --- a/.github/actions/docker-tag-push/retry_push.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Retry docker push with exponential backoff. -# Safe under `set -e`: the `if` conditional context prevents a failed -# `docker push` from triggering an immediate exit. -retry_push() { - local image="$1" - local max_attempts=3 - local wait_seconds=10 - local attempt=1 - - while true; do - if docker push "$image"; then - return 0 - fi - echo "Push failed for $image (attempt ${attempt}/${max_attempts})." >&2 - - if (( attempt >= max_attempts )); then - echo "Push failed after ${max_attempts} attempts: $image" >&2 - return 1 - fi - - echo "Retrying in ${wait_seconds}s..." - sleep "$wait_seconds" - attempt=$((attempt + 1)) - wait_seconds=$((wait_seconds * 2)) - if (( wait_seconds > 120 )); then - wait_seconds=120 - fi - done -} diff --git a/.github/scripts/retry_docker.sh b/.github/scripts/retry_docker.sh new file mode 100644 index 000000000..26da00524 --- /dev/null +++ b/.github/scripts/retry_docker.sh @@ -0,0 +1,45 @@ +# Retry docker operations with exponential backoff. +# Safe under `set -e`: the `if` conditional context prevents a failed +# `docker ` from triggering an immediate exit. +retry_docker_operation() { + local operation="$1" + local image="$2" + local max_attempts=3 + local wait_seconds=10 + local attempt=1 + + if [[ "$operation" != "push" && "$operation" != "pull" ]]; then + echo "Unsupported docker operation: $operation (expected: push|pull)" >&2 + return 2 + fi + + while true; do + if docker "$operation" "$image"; then + return 0 + fi + echo "Docker ${operation} failed for $image (attempt ${attempt}/${max_attempts})." >&2 + + if (( attempt >= max_attempts )); then + echo "Docker ${operation} failed after ${max_attempts} attempts: $image" >&2 + return 1 + fi + + echo "Retrying docker ${operation} in ${wait_seconds}s..." + sleep "$wait_seconds" + attempt=$((attempt + 1)) + wait_seconds=$((wait_seconds * 2)) + if (( wait_seconds > 120 )); then + wait_seconds=120 + fi + done +} + +retry_push() { + local image="$1" + retry_docker_operation push "$image" +} + +retry_pull() { + local image="$1" + retry_docker_operation pull "$image" +} diff --git a/.github/workflows/build-test-distribute-flavor.yml b/.github/workflows/build-test-distribute-flavor.yml index 58af83f56..50f5c8f57 100644 --- a/.github/workflows/build-test-distribute-flavor.yml +++ b/.github/workflows/build-test-distribute-flavor.yml @@ -304,9 +304,10 @@ jobs: - name: Pull relevant images shell: bash run: | + source ./.github/scripts/retry_docker.sh start_time=$(date +%s) - docker pull ${{ steps.calculate-target-tag.outputs.test_image }} - docker pull quay.io/minio/minio + retry_pull ${{ steps.calculate-target-tag.outputs.test_image }} + retry_pull quay.io/minio/minio end_time=$(date +%s) duration=$((end_time - start_time)) echo "⏱️ Image pull duration: ${duration}s" @@ -402,9 +403,10 @@ jobs: - name: Pull relevant images shell: bash run: | + source ./.github/scripts/retry_docker.sh start_time=$(date +%s) - docker pull ${{ steps.calculate-target-tag.outputs.test_image }} - docker pull quay.io/minio/minio + retry_pull ${{ steps.calculate-target-tag.outputs.test_image }} + retry_pull quay.io/minio/minio end_time=$(date +%s) duration=$((end_time - start_time)) echo "⏱️ Image pull duration: ${duration}s" diff --git a/.github/workflows/container-validation-dynamo.yml b/.github/workflows/container-validation-dynamo.yml index 373ba4a1f..526c9ceab 100644 --- a/.github/workflows/container-validation-dynamo.yml +++ b/.github/workflows/container-validation-dynamo.yml @@ -138,7 +138,9 @@ jobs: aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Pull image - run: docker pull ${{ env.IMAGE_TAG }} + run: | + source ./.github/scripts/retry_docker.sh + retry_pull ${{ env.IMAGE_TAG }} - name: Run Rust checks (block-manager + media-ffmpeg + integration tests) run: | docker run --rm --runtime=nvidia --gpus all --user root -w /workspace/lib/llm \ @@ -175,7 +177,9 @@ jobs: aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Pull image - run: docker pull ${{ env.IMAGE_TAG }} + run: | + source ./.github/scripts/retry_docker.sh + retry_pull ${{ env.IMAGE_TAG }} - name: Run pytest (parallel tests with xdist) uses: ./.github/actions/pytest with: @@ -205,7 +209,9 @@ jobs: aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} - name: Pull image - run: docker pull ${{ env.IMAGE_TAG }} + run: | + source ./.github/scripts/retry_docker.sh + retry_pull ${{ env.IMAGE_TAG }} - name: Run pytest (sequential tests) uses: ./.github/actions/pytest with: