[IE CLDNN] Set arguments once (#1699)

inference-engine/thirdparty/clDNN/common/khronos_ocl_clhpp/cl2_ext.hpp

@@ -662,6 +662,13 @@ typedef CL_API_ENTRY cl_mem(CL_API_CALL * PFN_clCreateFromMediaSurfaceINTEL)(
             fn = supports_usm ? load_entrypoint<PFN_clSetKernelArgMemPointerINTEL>(get(), "clSetKernelArgMemPointerINTEL") : nullptr;
         }
 
+        KernelIntel(const Kernel &other, PFN_clSetKernelArgMemPointerINTEL fn) : Kernel(other), fn(fn) { }
+
+        KernelIntel clone() const {
+            Kernel cloned_kernel(this->getInfo<CL_KERNEL_PROGRAM>(), this->getInfo<CL_KERNEL_FUNCTION_NAME>().c_str());
+            return KernelIntel(cloned_kernel, fn);
+        }
+
         cl_int setArgUsm(cl_uint index, const UsmMemory& mem) {
             if (!fn)
                 throw std::runtime_error("[CL ext] clSetKernelArgMemPointerINTEL function ptr is null. Can not set USM arg.");

inference-engine/thirdparty/clDNN/src/gpu/custom_gpu_primitive_gpu.cpp

@@ -51,16 +51,26 @@ struct custom_gpu_primitive_gpu : typed_primitive_impl<custom_gpu_primitive> {
                   arg.get_program().get_id(),
                   arg.get_program().get_engine().get_context()->get_configuration().dump_custom_program) {}
 
-    event_impl::ptr execute_impl(const std::vector<event_impl::ptr>& events,
-                                 custom_gpu_primitive_inst& instance) override {
+    void set_arguments_impl(custom_gpu_primitive_inst& instance) override {
         auto net_id = instance.get_network().get_id();
         gpu::kernel::kernel_arguments_data args;
         for (auto& dep : instance.dependencies()) {
             args.inputs.push_back((memory_impl::cptr) &(dep->output_memory()));
         }
         args.output = (memory_impl::cptr) &instance.output_memory();
+        _kernel.set_arguments(net_id, *cl_kernel.get(), args);
+    }
+
+    void cleanup_impl(custom_gpu_primitive_inst& instance) override {
+        auto net_id = instance.get_network().get_id();
+        _kernel.cleanup(net_id);
+    }
+
+    event_impl::ptr execute_impl(const std::vector<event_impl::ptr>& events,
+                                 custom_gpu_primitive_inst& instance) override {
+        auto net_id = instance.get_network().get_id();
         _kernel.set_output_event(net_id, instance.node.is_output());
-        return _kernel.run(net_id, *cl_kernel.get(), events, args);
+        return _kernel.run(net_id, *cl_kernel.get(), events);
     }
 };
 

inference-engine/thirdparty/clDNN/src/gpu/generic_layer_gpu.cpp

@@ -39,8 +39,8 @@ struct generic_layer_gpu : typed_primitive_impl<generic_layer> {
                   outer.get_primitive()->generic_params.clKernel->kernelString,
                   arg.get_program().get_id()) {}
 
-    event_impl::ptr execute_impl(const std::vector<event_impl::ptr>& events, generic_layer_inst& instance) override {
-        uint32_t net_id = instance.get_network().get_id();
+    void set_arguments_impl(generic_layer_inst& instance) override {
+        auto net_id = instance.get_network().get_id();
         gpu::kernel::kernel_arguments_data args;
         args.scalars = &_cl_kernel_data.scalars;
 
@@ -48,8 +48,18 @@ struct generic_layer_gpu : typed_primitive_impl<generic_layer> {
             args.inputs.push_back((memory_impl::cptr) &instance.input_memory(i));
         }
         args.output = (memory_impl::cptr) &instance.output_memory();
+        _kernel.set_arguments(net_id, _cl_kernel_data, args);
+    }
+
+    void cleanup_impl(generic_layer_inst& instance) override {
+        auto net_id = instance.get_network().get_id();
+        _kernel.cleanup(net_id);
+    }
+
+    event_impl::ptr execute_impl(const std::vector<event_impl::ptr>& events, generic_layer_inst& instance) override {
+        uint32_t net_id = instance.get_network().get_id();
         _kernel.set_output_event(net_id, instance.node.is_output());
-        return _kernel.run(net_id, _cl_kernel_data, events, args);
+        return _kernel.run(net_id, _cl_kernel_data, events);
     }
 };
 

inference-engine/thirdparty/clDNN/src/gpu/kernel.cpp

@@ -40,9 +40,9 @@ inline cl::NDRange toNDRange(const std::vector<size_t>& v) {
     }
 }
 
-void set_arguments(kernels_cache::kernel_type& kernel,
-                   const kernel_selector::kernel_arguments& args,
-                   const kernel::kernel_arguments_data& data) {
+void set_arguments_impl(kernels_cache::kernel_type& kernel,
+                        const kernel_selector::kernel_arguments& args,
+                        const kernel::kernel_arguments_data& data) {
     for (uint32_t i = 0; i < static_cast<uint32_t>(args.size()); i++) {
         cl_int status = CL_INVALID_ARG_VALUE;
         switch (args[i].t) {
@@ -244,21 +244,41 @@ void set_arguments(kernels_cache::kernel_type& kernel,
 }
 }  // namespace
 
-event_impl::ptr kernel::run(uint32_t queue_id,
-                            const kernel_selector::cl_kernel_data& kernel_data,
-                            const std::vector<event_impl::ptr>& dependencies,
-                            const kernel_arguments_data& args) const {
+void kernel::set_arguments(uint32_t queue_id,
+                           const kernel_selector::cl_kernel_data& kernel_data,
+                           const kernel_arguments_data& args) {
     static std::mutex m;
     std::lock_guard<std::mutex> guard(m);
-    auto clkernel = context()->get_kernels_cache(_prog_id).get_kernel(_kernel_id, _one_time_kernel);
+    auto compiled_kernel = context()->get_kernels_cache(_prog_id).get_kernel(_kernel_id, _one_time_kernel);
+
+    // Create a copy of cl kernel for each stream if it doesn't exist
+    // Copy is needed to avoid data races between streams, but we create it only once for each stream
+    // because the cloning is quite expensive.
+    // Mutex is still needed to ensure that insert operation into the map is thread safe
+    if (_cl_kernels.find(queue_id) == _cl_kernels.end())
+        _cl_kernels[queue_id] = compiled_kernel.clone();
+
     try {
-        set_arguments(clkernel, kernel_data.arguments, args);
+        set_arguments_impl(_cl_kernels.at(queue_id), kernel_data.arguments, args);
     } catch (cl::Error const& err) {
         throw ocl_error(err);
     }
+}
+
+void kernel::cleanup(uint32_t queue_id) {
+    _cl_kernels.erase(queue_id);
+}
+
+event_impl::ptr kernel::run(uint32_t queue_id,
+                            const kernel_selector::cl_kernel_data& kernel_data,
+                            const std::vector<event_impl::ptr>& dependencies) const {
+
+    if (_cl_kernels.find(queue_id) == _cl_kernels.end() || _cl_kernels.at(queue_id).get() == NULL) {
+        throw std::runtime_error("[clDNN] Kernel for layer " + kernel_data.layerID + " is not found for stream " + std::to_string(queue_id));
+    }
 
     return context()->enqueue_kernel(queue_id,
-                                     clkernel,
+                                     _cl_kernels.at(queue_id),
                                      toNDRange(kernel_data.workGroups.global),
                                      toNDRange(kernel_data.workGroups.local),
                                      dependencies);

inference-engine/thirdparty/clDNN/src/gpu/kernel.h

@@ -35,20 +35,26 @@ class kernel : public context_holder {
     bool _one_time_kernel;  // If this flag is true, the kernel is intended to be executed only once (can be removed
                             // later from the cache).
 
+    std::map<uint32_t, kernels_cache::kernel_type> _cl_kernels;
+
 public:
     explicit kernel(std::shared_ptr<gpu_toolkit> context,
                     const std::shared_ptr<kernel_selector::kernel_string>& kernel_string,
                     uint32_t prog_id,
                     bool dump_custom_program = false,
                     bool one_time_kernel = false)
-        : context_holder(context),
-          _prog_id(prog_id),
-          _kernel_id(
-              context->get_kernels_cache(prog_id).set_kernel_source(kernel_string, dump_custom_program, one_time_kernel)),
-          _one_time_kernel(one_time_kernel) {}
+        : context_holder(context)
+        , _prog_id(prog_id)
+        , _kernel_id(context->get_kernels_cache(prog_id).set_kernel_source(kernel_string, dump_custom_program, one_time_kernel))
+        , _one_time_kernel(one_time_kernel)
+        , _cl_kernels({}) {}
 
     kernel(const kernel& other)
-        : context_holder(other.context()), _prog_id(other._prog_id), _kernel_id(other._kernel_id), _one_time_kernel(other._one_time_kernel) {}
+        : context_holder(other.context())
+        , _prog_id(other._prog_id)
+        , _kernel_id(other._kernel_id)
+        , _one_time_kernel(other._one_time_kernel)
+        , _cl_kernels(other._cl_kernels) {}
 
     kernel& operator=(const kernel& other) {
         if (this == &other) {
@@ -58,6 +64,7 @@ public:
         _kernel_id = other._kernel_id;
         _prog_id = other._prog_id;
         _one_time_kernel = other._one_time_kernel;
+        _cl_kernels = other._cl_kernels;
 
         return *this;
     }
@@ -88,10 +95,13 @@ public:
         context()->set_output_event(net_id, is_out_event);
     }
 
+    void cleanup(uint32_t queue_id);
+    void set_arguments(uint32_t queue_id,
+                       const kernel_selector::cl_kernel_data& kernel_data,
+                       const kernel_arguments_data& args);
     event_impl::ptr run(uint32_t queue_id,
                         const kernel_selector::cl_kernel_data& kernel_data,
-                        const std::vector<event_impl::ptr>& dependencies,
-                        const kernel_arguments_data& args) const;
+                        const std::vector<event_impl::ptr>& dependencies) const;
 };
 
 }  // namespace gpu

inference-engine/thirdparty/clDNN/src/gpu/kernel_runner.cpp

@@ -214,7 +214,8 @@ std::vector<std::chrono::nanoseconds> kernel_runner::run_kernels(const kernel_se
             for (int iteration = 0; iteration < runs_per_kernel; iteration++) {
                 event_impl::ptr event;
                 try {
-                    event = kernels[i].run(0, it->kernels[0], {}, args);
+                    kernels[i].set_arguments(0, it->kernels[0], args);
+                    event = kernels[i].run(0, it->kernels[0], {});
                 } catch (std::exception& e) {
                     std::cout << "[clDNN] Could not run kernel for auto-tune: " << it->kernelName
                               << " with auto-tune index " << it->autoTuneIndex << std::endl

inference-engine/thirdparty/clDNN/src/gpu/primitive_gpu_base.h

@@ -118,6 +118,41 @@ protected:
         return events_waiter(_outer.get_program().get_engine().get_context()).run(net_id, events);
     }
 
+    void set_arguments_impl(typed_primitive_inst<PType>& instance) override {
+        uint32_t net_id = instance.get_network().get_id();
+        if (optimized_out(instance) || is_cpu()) {
+            return;
+        }
+
+        auto split = get_split();
+
+        // we iterate over split first in order to be able parallelism with OOOQ mechanism.
+        for (size_t k = 0; k < _kernels.size(); ++k) {
+            for (decltype(split) i = 0; i < split; i++) {
+                auto args = get_arguments(instance, i);
+                args.scalars = &_kernel_data.kernels[k].scalars;
+                args.split = i;
+
+                for (const auto& m : _intermediates_memory) {
+                    args.intermediates.push_back(m);
+                }
+
+                _kernels[k].set_arguments(net_id, _kernel_data.kernels[k], args);
+            }
+        }
+    }
+
+    void cleanup_impl(typed_primitive_inst<PType>& instance) override {
+        uint32_t net_id = instance.get_network().get_id();
+        if (optimized_out(instance) || is_cpu()) {
+            return;
+        }
+
+        for (size_t k = 0; k < _kernels.size(); ++k) {
+            _kernels[k].cleanup(net_id);
+        }
+    }
+
     event_impl::ptr execute_impl(const std::vector<event_impl::ptr>& events,
                                  typed_primitive_inst<PType>& instance) override {
         uint32_t net_id = instance.get_network().get_id();
@@ -134,14 +169,6 @@ protected:
         for (size_t k = 0; k < _kernels.size(); ++k) {
             std::vector<event_impl::ptr> new_events;
             for (decltype(split) i = 0; i < split; i++) {
-                auto args = get_arguments(instance, i);
-                args.scalars = &_kernel_data.kernels[k].scalars;
-                args.split = i;
-
-                for (const auto& m : _intermediates_memory) {
-                    args.intermediates.push_back(m);
-                }
-
                 // is any user of the prim's users is an detecion output, set prim as a output event (event won't be
                 // nullptr)
                 auto users = instance.node.get_users();
@@ -152,7 +179,7 @@ protected:
                     _kernels[k].set_output_event(net_id, instance.node.is_output());
                 }
 
-                auto event = _kernels[k].run(net_id, _kernel_data.kernels[k], tmp_events, args);
+                auto event = _kernels[k].run(net_id, _kernel_data.kernels[k], tmp_events);
                 new_events.push_back(event);
             }
 

inference-engine/thirdparty/clDNN/src/gpu/wait_for_events_gpu.cpp

@@ -32,6 +32,9 @@ class wait_for_events_gpu : public primitive_impl {
 public:
     explicit wait_for_events_gpu(const program_node& /*node*/) {}
 
+    void set_arguments(primitive_inst& /*instance*/) override {}
+    void cleanup(primitive_inst& /*instance*/) override {}
+
     event_impl::ptr execute(const std::vector<event_impl::ptr>& events, primitive_inst& instance) override {
         uint32_t net_id = instance.get_network().get_id();
         events_waiter events_waiter(instance.get_network().get_engine().get_context());

inference-engine/thirdparty/clDNN/src/include/network_impl.h

@@ -76,6 +76,7 @@ public:
     const program_impl::graph_optimizer_info& get_optimizer_passes_info() const;
     void execute(const std::vector<event_impl::ptr>& events);
     void validate_primitives();
+    void set_arguments();
     // Implementation specific calls
     std::shared_ptr<primitive_inst> get_primitive(const primitive_id& id);
     std::string get_primitive_info(const primitive_id& id) const;
@@ -97,6 +98,7 @@ private:
     const program_impl::cptr _program;
     uint16_t _stream_id;
     bool _internal;
+    bool _reset_arguments;
     float _learning_rate = static_cast<float>(0.00001);
 
     std::map<primitive_id, std::shared_ptr<primitive_inst>> _primitives;

inference-engine/thirdparty/clDNN/src/include/primitive_inst.h

@@ -55,6 +55,8 @@ struct primitive_impl {
         : _weights_reorder_params(params), _kernel_name(kernel_name) {}
     virtual ~primitive_impl() = default;
 
+    virtual void set_arguments(primitive_inst& instance) = 0;
+    virtual void cleanup(primitive_inst& instance) = 0;
     virtual event_impl::ptr execute(const std::vector<event_impl::ptr>& events, primitive_inst& instance) = 0;
     virtual bool validate(const primitive_inst& instance) const = 0;
     std::string get_kernel_name() const { return _kernel_name; }
@@ -107,6 +109,8 @@ public:
     }
 
     event_impl::ptr execute(const std::vector<event_impl::ptr>& events);
+    void set_arguments();
+    void cleanup();
     bool validate() const {
         if (_impl == nullptr)
             throw std::invalid_argument("[Internal cldnn error].  Validation method for nullptr impl is not allowed.");
@@ -125,6 +129,18 @@ public:
     size_t get_fused_mem_count() const { return _node.get_fused_inputs_count(); }
     size_t get_fused_mem_offset() const { return _node.get_fused_primitives()[0].dep_start_idx; }
 
+    bool has_mutable_input() const {
+        return _has_mutable_input;
+    }
+
+    void set_mutable_input(bool val) {
+        _has_mutable_input = val;
+    }
+
+    bool is_output() const {
+        return _node.is_output();
+    }
+
 protected:
     primitive_inst(network_impl& network, program_node const& node, bool allocate_memory);
 
@@ -154,6 +170,7 @@ protected:
     bool _output_changed;  // todo: implement output reuse if neither of inputs has changed
     bool _has_valid_input =
         true;  // by default all primitives has valid inputs, exception is input_layout (see input_layout_inst)
+    bool _has_mutable_input = false;
 
     memory_impl::ptr allocate_output();
     static std::vector<std::shared_ptr<primitive_inst>> build_exec_deps(
@@ -189,6 +206,29 @@ private:
 
         return execute_impl(event, reinterpret_cast<typed_primitive_inst<PType>&>(instance));
     }
+
+    void set_arguments(primitive_inst& instance) override {
+        if (instance.type() != PType::type_id())
+            throw std::invalid_argument("Implementation type does not match primitive type");
+        if (instance.get_impl() != this)
+            throw std::invalid_argument(
+                "Trying to set_arguments for primitive implementation with mismatching primitive instance");
+
+        return set_arguments_impl(reinterpret_cast<typed_primitive_inst<PType>&>(instance));
+    }
+
+    void cleanup(primitive_inst& instance) override {
+        if (instance.type() != PType::type_id())
+            throw std::invalid_argument("Implementation type does not match primitive type");
+        if (instance.get_impl() != this)
+            throw std::invalid_argument(
+                "Trying to cleanup primitive implementation with mismatching primitive instance");
+
+        return cleanup_impl(reinterpret_cast<typed_primitive_inst<PType>&>(instance));
+    }
+
+    virtual void set_arguments_impl(typed_primitive_inst<PType>& /*instance*/) {};
+    virtual void cleanup_impl(typed_primitive_inst<PType>& /*instance*/) {};
     virtual event_impl::ptr execute_impl(const std::vector<event_impl::ptr>& event,
                                          typed_primitive_inst<PType>& instance) = 0;
 

inference-engine/thirdparty/clDNN/src/network.cpp

@@ -286,7 +286,7 @@ Network_impl will always have net_id = 0 when it will be cldnn internal micronet
 opt pass).
 */
 network_impl::network_impl(const program_impl& program, uint16_t stream_id, bool is_internal)
-    : _program(&program), _stream_id(stream_id), _internal(is_internal) {
+    : _program(&program), _stream_id(stream_id), _internal(is_internal), _reset_arguments(true) {
     static std::atomic<uint32_t> id_gen{0};
     if (!_internal) {
         net_id = ++id_gen;
@@ -304,6 +304,10 @@ network_impl::network_impl(const program_impl& program, uint16_t stream_id, bool
 }
 
 network_impl::~network_impl() {
+    for (auto const& prim : _exec_order) {
+        prim->cleanup();
+    }
+
     auto toolkit = get_engine().get_context();
     get_engine().get_memory_pool().clear_pool_for_network(net_id);
     toolkit->release_pending_memory(net_id);
@@ -332,6 +336,16 @@ void network_impl::validate_primitives() {
     }
 }
 
+void network_impl::set_arguments() {
+    if (!_reset_arguments)
+        return;
+
+    for (auto const& prim : _exec_order) {
+        prim->set_arguments();
+    }
+    _reset_arguments = false;
+}
+
 void network_impl::reset_execution(bool wait) {
     if (wait && _events.size() > 0) {
         std::vector<event_impl::ptr> events;
@@ -504,6 +518,8 @@ void network_impl::execute(const std::vector<refcounted_obj_ptr<event_impl>>& ev
     cl_int err;
     cl::SharedSurfLock lock(get_engine().get_context()->queue(get_id()).get(), surfaces, &err);
 
+    set_arguments();
+
     for (auto& inst : _exec_order) {
 #ifdef DEBUG_DUMP_PATH
         auto& node = _program->get_node(inst->id());
@@ -524,6 +540,12 @@ void network_impl::execute(const std::vector<refcounted_obj_ptr<event_impl>>& ev
         }
 #endif
 #endif
+
+        // If a node has mutable input or it's an output, then the input/output buffers might be changed
+        // So we need to set arguments on each execution.
+        if (inst->has_mutable_input() || inst->is_output()) {
+            inst->set_arguments();
+        }
         execute_primitive(inst, events);
 #ifdef DEBUG_DUMP_PATH
 #if DUMP_SINGLE_LAYER
@@ -705,6 +727,13 @@ void network_impl::allocate_primitive_instance(program_node const& node) {
         return;
 
     auto inst = node.type()->create_instance(*this, node);
+    for (auto& dep : node.get_dependencies()) {
+        if (dep->is_type<input_layout>() || dep->is_type<mutable_data>() || dep->can_be_optimized()) {
+            inst->set_mutable_input(true);
+            break;
+        }
+    }
+
     _primitives[node.id()] = inst;
     if (node.is_input())
         _inputs.push_back(inst);

inference-engine/thirdparty/clDNN/src/primitive_inst.cpp

@@ -109,6 +109,20 @@ event_impl::ptr primitive_inst::execute(const std::vector<event_impl::ptr>& even
     return _impl->execute(dependencies, *this);
 }
 
+void primitive_inst::set_arguments() {
+    const auto primitive_id = id();
+    CLDNN_ERROR_BOOL(primitive_id,
+                     "Invalid/unset input",
+                     !_has_valid_input,
+                     "Cannot set arguments for primitive " + primitive_id + " with invalid/unset input");
+
+    _impl->set_arguments(*this);
+}
+
+void primitive_inst::cleanup() {
+    _impl->cleanup(*this);
+}
+
 void primitive_inst::build_deps() {
     if (_deps.empty() && !_node.get_dependencies().empty()) {
         _deps = _network.get_primitives(_node.get_dependencies());