[GPU] Return false when dynamic shape in check_allocatable() to aviod asstion by big upper bound (#23895)

### Details:
- Return false when dynamic shape in check_allocatable() to aviod
asstion by big upper bound
 - Add excessive upper bound test case in softmax dynamic shape test

### Tickets:
 - 135307

src/plugins/intel_gpu/src/runtime/ocl/ocl_engine.cpp

@@ -128,21 +128,37 @@ allocation_type ocl_engine::detect_usm_allocation_type(const void* memory) const
 bool ocl_engine::check_allocatable(const layout& layout, allocation_type type) {
     OPENVINO_ASSERT(supports_allocation(type) || type == allocation_type::cl_mem, "[GPU] Unsupported allocation type: ", type);
 
-    OPENVINO_ASSERT(layout.bytes_count() <= get_device_info().max_alloc_mem_size,
+    bool exceed_allocatable_mem_size = (layout.bytes_count() > get_device_info().max_alloc_mem_size);
+
+    // When dynamic shape upper bound makes bigger buffer, then return false.
+    if (exceed_allocatable_mem_size && layout.is_dynamic()) {
+        OPENVINO_ASSERT(layout.has_upper_bound(), "[GPU] Dynamic shape without upper bound tries to allocate");
+        return false;
+    }
+
+    OPENVINO_ASSERT(!exceed_allocatable_mem_size,
                     "[GPU] Exceeded max size of memory object allocation: ",
                     "requested ", layout.bytes_count(), " bytes, "
                     "but max alloc size supported by device is ", get_device_info().max_alloc_mem_size, " bytes.",
                     "Please try to reduce batch size or use lower precision.");
 
     auto used_mem = get_used_device_memory(allocation_type::usm_device) + get_used_device_memory(allocation_type::usm_host);
+    auto exceed_available_mem_size = (layout.bytes_count() + used_mem > get_max_memory_size());
+
+    // When dynamic shape upper bound makes bigger buffer, then return false.
+    if (exceed_available_mem_size && layout.is_dynamic()) {
+        OPENVINO_ASSERT(layout.has_upper_bound(), "[GPU] Dynamic shape without upper bound tries to allocate");
+        return false;
+    }
+
 #ifdef __unix__
     // Prevent from being killed by Ooo Killer of Linux
-    OPENVINO_ASSERT(layout.bytes_count() + used_mem <= get_max_memory_size(),
+    OPENVINO_ASSERT(!exceed_available_mem_size,
                     "[GPU] Exceeded max size of memory allocation: ",
                     "Required ", layout.bytes_count(), " bytes, already occupied : ", used_mem, " bytes, ",
                     "but available memory size is ", get_max_memory_size(), " bytes");
 #else
-    if (layout.bytes_count() + used_mem > get_max_memory_size()) {
+    if (exceed_available_mem_size) {
         GPU_DEBUG_COUT << "[Warning] [GPU] Exceeded max size of memory allocation: " << "Required " << layout.bytes_count() << " bytes, already occupied : "
                        << used_mem << " bytes, but available memory size is " << get_max_memory_size() << " bytes" << std::endl;
         GPU_DEBUG_COUT << "Please note that performance might drop due to memory swap." << std::endl;

src/plugins/intel_gpu/tests/functional/dynamic_tests/gpu_dyn_huge_input_range.cpp

@@ -190,8 +190,6 @@ protected:
 
 TEST_P(DynamicShapeHugeRangeGPUTest, Inference) {
     run();
-    if (!exception)
-        FAIL() << "This test case is checking the exception that the object allocation is larger than the max_alloc_mem_size.";
 }
 
 std::map<std::string, std::string> emptyAdditionalConfig;

src/plugins/intel_gpu/tests/functional/single_layer_tests/dynamic/softmax.cpp

@@ -100,7 +100,11 @@ const std::vector<ov::test::InputShape> inputShapes4D = {
     {
         {{20, 100}, 12, {1, 400}, {1, 300}},
         {{30, 12, 10, 20}, {20, 12, 5, 40}}
-    }
+    },
+    { // Excessive upper bound case
+        {3, 8, {128, 16384}, {128, 16384}},
+        {{3, 8, 128, 128}}
+    },
 };
 
 const std::vector<int64_t> axis5D = {0, 1, 2, 4};