diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp
index 66627f58e63..63d73f7c090 100644
--- a/src/inference/src/dev/core_impl.cpp
+++ b/src/inference/src/dev/core_impl.cpp
@@ -1353,11 +1353,9 @@ void ov::CoreImpl::set_property_for_device(const ov::AnyMap& configMap, const st
             {
                 OPENVINO_SUPPRESS_DEPRECATED_START
                 if (device_supports_cache_dir(plugin.second)) {
-                    ov::AnyMap empty_map;
-                    auto cacheConfig = coreConfig.get_cache_config_for_device(plugin.second, empty_map);
-                    if (cacheConfig._cacheManager) {
-                        configCopy[ov::cache_dir.name()] = cacheConfig._cacheDir;
-                    }
+                    ov::AnyMap empty_map = {};
+                    configCopy[ov::cache_dir.name()] =
+                        coreConfig.get_cache_config_for_device(plugin.second, empty_map)._cacheDir;
                 } else if (configCopy.count(ov::cache_dir.name()) > 0) {
                     // Remove "CACHE_DIR" from config if it is not supported by plugin
                     configCopy.erase(ov::cache_dir.name());
diff --git a/src/plugins/proxy/tests/proxy_tests.cpp b/src/plugins/proxy/tests/proxy_tests.cpp
index 0343e40ee24..fc9d34e372a 100644
--- a/src/plugins/proxy/tests/proxy_tests.cpp
+++ b/src/plugins/proxy/tests/proxy_tests.cpp
@@ -664,7 +664,7 @@ void ov::proxy::tests::ProxyTests::register_plugin_support_subtract(ov::Core& co
             for (const auto& it : properties) {
                 if (it.first == ov::enable_profiling.name())
                     m_profiling = it.second.as<bool>();
-                else if (it.first == ov::device::id.name())
+                else if (it.first == ov::device::id.name() || it.first == ov::cache_dir.name())
                     continue;
                 else
                     OPENVINO_THROW(get_device_name(), " set config: " + it.first);