From 1ad5aa6c34a33322eae51388b78e6170a81dc208 Mon Sep 17 00:00:00 2001
From: Fazzie <1240419984@qq.com>
Date: Wed, 10 Nov 2021 09:47:51 +0800
Subject: [PATCH 1/3] enable opengl Texture

---
 compile.sh                                        | 15 +++++++++++++++
 include/api/context.h                             | 10 ++++++++++
 include/api/data_type.h                           |  1 +
 include/api/model.h                               | 15 +++++++++++++++
 mindspore/lite/include/context.h                  |  2 --
 mindspore/lite/src/CMakeLists.txt                 |  5 +++++
 mindspore/lite/src/cxx_api/converters.cc          |  2 --
 mindspore/lite/src/lite_session.cc                |  1 +
 mindspore/lite/src/lite_session.h                 |  1 -
 .../lite/src/runtime/gpu/opencl/opencl_runtime.cc |  3 ++-
 .../lite/src/runtime/gpu/opencl/opencl_wrapper.cc |  1 +
 .../src/runtime/kernel/opencl/kernel/gl_to_cl.cc  |  1 +
 .../src/runtime/kernel/opencl/kernel/gl_to_cl.h   |  8 ++++++++
 13 files changed, 59 insertions(+), 6 deletions(-)
 create mode 100755 compile.sh

diff --git a/compile.sh b/compile.sh
new file mode 100755
index 0000000000..e0c09fb479
--- /dev/null
+++ b/compile.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+./build.sh -I arm64 -j 4 -i
+cd output
+tar -zxvf mindspore-lite-1.5.0-android-aarch64.tar.gz
+cd mindspore-lite-1.5.0-android-aarch64/
+adb push ./tools/benchmark/benchmark ./runtime/lib/libmindspore-lite.so /data/local/tmp/mql
+
+function run_adb_cmd() {
+  export MINDSPORE_DUMP_CONFIG=/data/local/tmp/mql/data_dump.json
+  #adb shell chmod 0777 /data/local/tmp/mql/benchmark
+  #adb shell " export MINDSPORE_DUMP_CONFIG=/data/local/tmp/mql/dump.json
+  adb shell "export LD_LIBRARY_PATH=/data/local/tmp/mql && cd /data/local/tmp/mql && ./benchmark --modelFile=./mtk_face_features_v1.ms  --enableGLTexture=true --device=GPU"
+}
+
+run_adb_cmd
\ No newline at end of file
diff --git a/include/api/context.h b/include/api/context.h
index 420eae067b..ecb0a8fc4e 100644
--- a/include/api/context.h
+++ b/include/api/context.h
@@ -257,6 +257,16 @@ class MS_API GPUDeviceInfo : public DeviceInfoContext {
   /// \return Whether enable float16 inference.
   bool GetEnableFP16() const;
 
+  /// \brief Set enables to sharing mem with OpenGL
+  ///
+  /// \param[in] is_enable_sharing_mem_with_gl Enable sharing OpenCL Memory with OpenGL or not.
+  void SetEnableGLTexture(bool is_enable_gl_texture);
+
+  /// \brief Get enables to sharing mem with OpenGL
+  ///
+  /// \return Whether enable sharing mem with OpenGL.
+  bool GetEnableGLTexture() const;
+
  private:
   void SetPrecisionMode(const std::vector<char> &precision_mode);
   std::vector<char> GetPrecisionModeChar() const;
diff --git a/include/api/data_type.h b/include/api/data_type.h
index 053244241b..134490656f 100644
--- a/include/api/data_type.h
+++ b/include/api/data_type.h
@@ -38,6 +38,7 @@ enum class DataType : int {
   kNumberTypeFloat16 = 42,
   kNumberTypeFloat32 = 43,
   kNumberTypeFloat64 = 44,
+  kNumberTypeGLUInt = 45,
   kNumberTypeEnd = 46,
   // add new enum here
   kInvalidType = INT32_MAX,
diff --git a/include/api/model.h b/include/api/model.h
index 8de66fa0d8..a9922e3812 100644
--- a/include/api/model.h
+++ b/include/api/model.h
@@ -29,6 +29,11 @@
 #include "include/api/cell.h"
 #include "include/api/cfg.h"
 #include "include/api/dual_abi_helper.h"
+#ifdef ENABLE_OPENGL_TEXTURE
+#include <EGL/egl.h>
+#include <GLES3/gl3.h>
+#include <GLES3/gl32.h>
+#endif
 
 namespace mindspore {
 class ModelImpl;
@@ -181,6 +186,16 @@ class MS_API Model {
   /// \return The vector of output MSTensor.
   inline std::vector<MSTensor> GetOutputsByNodeName(const std::string &node_name);
 
+#ifdef ENABLE_OPENGL_TEXTURE
+  /// \brief Bind GLTexture2D object to cl Memory.
+  ///
+  /// \param[in] inputGlTexture The input GLTexture id for Model.
+  /// \param[in] outputGLTexture The output GLTexture id for Model.
+  ///
+  /// \return Status of operation.
+  Status BindGLTexture2DMemory(std::map<std::string, GLuint> &inputGlTexture, std::map<std::string, GLuint> *outputGLTexture);
+#endif
+
   /// \brief Inference model.
   ///
   /// \param[in] device_type Device type，options are kGPU, kAscend910, etc.
diff --git a/mindspore/lite/include/context.h b/mindspore/lite/include/context.h
index e7649e8ee5..6762bf0592 100644
--- a/mindspore/lite/include/context.h
+++ b/mindspore/lite/include/context.h
@@ -34,9 +34,7 @@ typedef struct GpuDeviceInfo {
   uint32_t gpu_device_id_ = 0;
   int rank_id_ = 0;
   int group_size_ = 0;
-#ifdef ENABLE_OPENGL_TEXTURE
   bool enable_gl_texture_ = false; /**<enable sharing OpenGL texture with OpenCL */
-#endif
 } GpuDeviceInfo;
 
 /// \brief NpuDeviceInfo defined for NPU's configuration information.
diff --git a/mindspore/lite/src/CMakeLists.txt b/mindspore/lite/src/CMakeLists.txt
index 3728f2c2ed..75b3bdceee 100644
--- a/mindspore/lite/src/CMakeLists.txt
+++ b/mindspore/lite/src/CMakeLists.txt
@@ -344,6 +344,11 @@ if(MSLITE_GPU_BACKEND STREQUAL opencl)
     add_subdirectory(runtime/kernel/opencl)
     target_link_libraries(mindspore-lite opencl_kernel_mid)
     target_link_libraries(mindspore-lite_static opencl_kernel_mid)
+    if(MSLITE_ENABLE_SHARING_MEM_WITH_OPENGL)
+        list(APPEND opengl_lib GLESv3 EGL)
+        target_link_libraries(mindspore-lite ${opengl_lib})
+        target_link_libraries(mindspore-lite_static ${opengl_lib})
+    endif ()
 elseif(MSLITE_GPU_BACKEND STREQUAL cuda)
     add_subdirectory(runtime/kernel/cuda)
     target_link_libraries(mindspore-lite cuda_kernel_mid)
diff --git a/mindspore/lite/src/cxx_api/converters.cc b/mindspore/lite/src/cxx_api/converters.cc
index 138a9b12b1..62ed373f1b 100644
--- a/mindspore/lite/src/cxx_api/converters.cc
+++ b/mindspore/lite/src/cxx_api/converters.cc
@@ -57,8 +57,6 @@ Status AddCpuDevice(const Context *a_context, lite::InnerContext *l_context, Dev
 Status AddGpuDevice(lite::InnerContext *l_context, DeviceInfoContext *device) {
   lite::DeviceInfo device_info = {0};
   auto gpu_context = device->Cast<GPUDeviceInfo>();
-  device_info.gpu_device_info_ = {gpu_context->GetEnableFP16(), gpu_context->GetDeviceID(), gpu_context->GetRankID(),
-                                  gpu_context->GetGroupSize()};
 #ifdef ENABLE_OPENGL_TEXTURE
   device_info.gpu_device_info_ = {gpu_context->GetEnableFP16(), gpu_context->GetDeviceID(), gpu_context->GetRankID(),
                                   gpu_context->GetGroupSize(), gpu_context->GetEnableGLTexture()};
diff --git a/mindspore/lite/src/lite_session.cc b/mindspore/lite/src/lite_session.cc
index 9033557e57..a677886404 100644
--- a/mindspore/lite/src/lite_session.cc
+++ b/mindspore/lite/src/lite_session.cc
@@ -535,6 +535,7 @@ int LiteSession::CompileGraph(Model *model) {
   InitGraphOutputTensors(model);
 
   // scheduler kernels
+
   Scheduler scheduler(context_, ms_context_, model, &tensors_, inputs_, outputs_, is_train_session_, &is_infershape_,
                       &is_control_flow_, execution_plan_, delegate_, delegate_device_type_);
   scheduler.SetupSchedulerCb(std::move(sched_cb_));
diff --git a/mindspore/lite/src/lite_session.h b/mindspore/lite/src/lite_session.h
index 813d44612d..e6e9252c99 100644
--- a/mindspore/lite/src/lite_session.h
+++ b/mindspore/lite/src/lite_session.h
@@ -80,7 +80,6 @@ class LiteSession : public session::LiteSession {
 
 #ifdef ENABLE_OPENGL_TEXTURE
   int BindGLTexture2DMemory(const std::map<std::string, GLuint> &inputGLTexture,
-                            std::map<std::string, GLuint> *outputGLTexture) override;
 #endif
   int Resize(const std::vector<mindspore::tensor::MSTensor *> &inputs,
              const std::vector<std::vector<int>> &dims) override;
diff --git a/mindspore/lite/src/runtime/gpu/opencl/opencl_runtime.cc b/mindspore/lite/src/runtime/gpu/opencl/opencl_runtime.cc
index 6c2c1f4627..176c59a32b 100644
--- a/mindspore/lite/src/runtime/gpu/opencl/opencl_runtime.cc
+++ b/mindspore/lite/src/runtime/gpu/opencl/opencl_runtime.cc
@@ -46,7 +46,6 @@ OpenCLRuntime *OpenCLRuntime::GetInstance() {
   static OpenCLRuntime ocl_runtime;
   if (instance_count_ == 0) {
     ocl_runtime_instance_ = &ocl_runtime;
-    ocl_runtime_instance_->Init();
   }
   instance_count_++;
   return ocl_runtime_instance_;
@@ -387,9 +386,11 @@ GpuInfo OpenCLRuntime::GetGpuInfo() { return gpu_info_; }
 
 bool OpenCLRuntime::GetFp16Enable() const { return fp16_enable_; }
 
+
 #ifdef ENABLE_OPENGL_TEXTURE
 bool OpenCLRuntime::GetGLTextureEnable() const { return enable_gl_texture_; }
 #endif
+bool OpenCLRuntime::GetGLTextureEnable() const { return enable_gl_texture_; }
 
 // if support fp16, set fp16 will success.
 bool OpenCLRuntime::SetFp16Enable(bool enable) {
diff --git a/mindspore/lite/src/runtime/gpu/opencl/opencl_wrapper.cc b/mindspore/lite/src/runtime/gpu/opencl/opencl_wrapper.cc
index 0520b2186f..01a348c9ed 100644
--- a/mindspore/lite/src/runtime/gpu/opencl/opencl_wrapper.cc
+++ b/mindspore/lite/src/runtime/gpu/opencl/opencl_wrapper.cc
@@ -693,6 +693,7 @@ cl_mem clCreateFromGLTexture(cl_context context, cl_mem_flags flags, cl_GLenum t
   MS_ASSERT(func != nullptr);
   return func(context, flags, target, miplevel, texture, errcode_ret);
 }
+
 #endif
 #endif
 
diff --git a/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.cc b/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.cc
index c2df1c1030..6a56c6ecee 100644
--- a/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.cc
+++ b/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.cc
@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 #ifdef ENABLE_OPENGL_TEXTURE
+
 #include "src/runtime/kernel/opencl/kernel/gl_to_cl.h"
 #include <map>
 #include <string>
diff --git a/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.h b/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.h
index df7dea3ce0..a781141eca 100644
--- a/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.h
+++ b/mindspore/lite/src/runtime/kernel/opencl/kernel/gl_to_cl.h
@@ -49,6 +49,10 @@ class GLToCLOpenCLKernel : public OpenCLKernel {
   cl_mem_flags flags_{CL_MEM_READ_ONLY};
   cl_GLenum target_{GL_TEXTURE_2D};
   cl_GLint mip_level_{0};
+<<<<<<< HEAD
+=======
+  cl_int *err_ = nullptr;
+>>>>>>> enable opengl Texture
   size_t N_{1};
   size_t H_{1};
   size_t W_{1};
@@ -56,5 +60,9 @@ class GLToCLOpenCLKernel : public OpenCLKernel {
 };
 }  // namespace mindspore::kernel
 
+<<<<<<< HEAD
 #endif
 #endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_OPENCL_KERNEL_GL_TO_CL_H_
+=======
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_OPENCL_KERNEL_GL_TO_CL_H_
+>>>>>>> enable opengl Texture
-- 
2.25.1