yankf21
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add rotate op for EXIF image info

This commit is contained in:
xulei2020 2020-12-01 16:56:10 +08:00
parent 44ea3902b8
commit ada57d0e58
11 changed files with 406 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
mindspore/ccsrc/minddata/dataset/kernels/image/exif_utils.cc Normal file
View File

@ -0,0 +1,128 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/kernels/image/exif_utils.h"
#include <algorithm>
#include <cstdint>
#define UNKNOW_ORIENTATION 0
namespace mindspore {
namespace dataset {
template <typename T>
T parse_bytes(const uint8_t *buf, bool intel_align);
template <>
uint8_t parse_bytes(const uint8_t *buf, bool intel_align) {
return *buf;
}
template <>
uint16_t parse_bytes(const uint8_t *buf, bool intel_align) {
uint16_t res;
if (intel_align) {
res = (static_cast<uint16_t>(buf[1]) << 8) | buf[0];
} else {
res = (static_cast<uint16_t>(buf[0]) << 8) | buf[1];
}
return res;
}
template <>
uint32_t parse_bytes(const uint8_t *buf, bool intel_align) {
uint32_t res;
if (intel_align) {
res = (static_cast<uint32_t>(buf[3]) << 24) | (static_cast<uint32_t>(buf[2]) << 16) |
(static_cast<uint32_t>(buf[1]) << 8) | buf[0];
} else {
res = (static_cast<uint32_t>(buf[0]) << 24) | (static_cast<uint32_t>(buf[1]) << 16) |
(static_cast<uint32_t>(buf[2]) << 8) | buf[3];
}
return res;
}
int parseExif(const uint8_t *buf, uint32_t len) {
bool intel_align = true;
uint32_t offset = 0;
if (!buf || len < 6) return UNKNOW_ORIENTATION;
if (!std::equal(buf, buf + 6, "Exif\0\0")) return UNKNOW_ORIENTATION;
offset += 6;
if (offset + 8 > len) return UNKNOW_ORIENTATION;
if (buf[offset] == 'I' && buf[offset + 1] == 'I') {
intel_align = true;
} else {
if (buf[offset] == 'M' && buf[offset + 1] == 'M')
intel_align = false;
else
return UNKNOW_ORIENTATION;
}
offset += 2;
if (parse_bytes<uint16_t>(buf + offset, intel_align) != 0x2a) return UNKNOW_ORIENTATION;
offset += 2;
uint32_t first_ifd_offset = parse_bytes<uint32_t>(buf + offset, intel_align);
offset += first_ifd_offset - 4;
if (offset >= len) return UNKNOW_ORIENTATION;
if (offset + 2 > len) return UNKNOW_ORIENTATION;
int num_entries = parse_bytes<uint16_t>(buf + offset, intel_align);
if (offset + 6 + 12 * num_entries > len) return UNKNOW_ORIENTATION;
offset += 2;
while (num_entries > 0) {
uint16_t tag = parse_bytes<uint16_t>(buf + offset, intel_align);
if (tag == 0x112) {
uint16_t format = parse_bytes<uint16_t>(buf + offset + 2, intel_align);
uint32_t length = parse_bytes<uint32_t>(buf + offset + 4, intel_align);
if (format == 3 && length) {
uint16_t orient = parse_bytes<uint16_t>(buf + offset + 8, intel_align);
return static_cast<int>(orient);
}
}
offset += 12;
num_entries--;
}
return UNKNOW_ORIENTATION;
}
int ExifInfo::parseOrientation(const unsigned char *data, unsigned len) {
if (!data || len < 4) return UNKNOW_ORIENTATION;
if (data[0] != 0xFF || data[1] != 0xD8) return UNKNOW_ORIENTATION;
while (len > 2) {
if (data[len - 1] == 0xD9 && data[len - 2] == 0xFF) break;
len--;
}
if (len <= 2) return UNKNOW_ORIENTATION;
unsigned int offset = 0;
for (; offset < len - 1; offset++) {
if (data[offset] == 0xFF && data[offset + 1] == 0xE1) break;
}
if (offset + 4 > len) return UNKNOW_ORIENTATION;
offset += 2;
uint16_t section_length = parse_bytes<uint16_t>(data + offset, false);
if (offset + section_length > len || section_length < 16) return UNKNOW_ORIENTATION;
offset += 2;
return parseExif(data + offset, len - offset);
}
} // namespace dataset
} // namespace mindspore

29
mindspore/ccsrc/minddata/dataset/kernels/image/exif_utils.h Normal file
View File

@ -0,0 +1,29 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_EXIF_H
#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_EXIF_H
namespace mindspore {
namespace dataset {
class ExifInfo {
public:
int parseOrientation(const unsigned char *data, unsigned len);
};
} // namespace dataset
} // namespace mindspore
#endif

13
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc
View File

@ -16,6 +16,7 @@
#include "minddata/dataset/kernels/image/lite_cv/image_process.h"
#include <limits.h>
#include <string.h>
#include <cmath>
#include <vector>
@ -61,6 +62,9 @@ static void ResizeBilinear3C(const unsigned char *src, int src_width, int src_he
double scale_width = static_cast<double>(src_width) / dst_width;
double scale_height = static_cast<double>(src_height) / dst_height;
if (dst_height >= (INT_MAX / 2 - dst_width)) {
return;
}
int *data_buf = new int[2 * dst_width + 2 * dst_height];
int *x_offset = data_buf;
@ -140,6 +144,9 @@ static void ResizeBilinear1C(const unsigned char *src, int src_width, int src_he
double scale_width = static_cast<double>(src_width) / dst_width;
double scale_height = static_cast<double>(src_height) / dst_height;
if (dst_height >= (INT_MAX / 2 - dst_width)) {
return;
}
int *data_buf = new int[2 * dst_width + 2 * dst_height];
int *x_offset = data_buf;
@ -408,7 +415,9 @@ bool ConvertTo(const LiteMat &src, LiteMat &dst, double scale) {
if (src.data_type_ != LDataType::UINT8) {
return false;
}
if (scale < 0.0 || scale > 100) {
return false;
}
(void)dst.Init(src.width_, src.height_, src.channel_, LDataType::FLOAT32);
const unsigned char *src_start_p = src;
float *dst_start_p = dst;
@ -442,7 +451,7 @@ bool Crop(const LiteMat &src, LiteMat &dst, int x, int y, int w, int h) {
if (x < 0 || y < 0 || w <= 0 || h <= 0) {
return false;
}
if (y + h > src.height_ || x + w > src.width_) {
if (y > src.height_ - h || x > src.width_ - w) {
return false;
}

5
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc
View File

@ -15,8 +15,8 @@
*/
#include "minddata/dataset/kernels/image/lite_cv/lite_mat.h"
#include <limits.h>
#include <algorithm>
#include <limits>
#include <cmath>
#ifdef ENABLE_ANDROID
#if defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64)
@ -252,6 +252,9 @@ void LiteMat::Release() {
void *LiteMat::AlignMalloc(unsigned int size) {
unsigned int length = sizeof(void *) + ALIGN - 1;
if (size > INT_MAX - length) {
return nullptr;
}
void *p_raw = reinterpret_cast<void *>(malloc(size + length));
if (p_raw) {
void **p_algin = reinterpret_cast<void **>(((size_t)(p_raw) + length) & ~(ALIGN - 1));

149
mindspore/ccsrc/minddata/dataset/kernels/image/lite_image_utils.cc
View File

@ -441,5 +441,154 @@ Status Pad(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output
return Status::OK();
}
static Status RotateAngleWithOutMirror(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output,
const uint64_t orientation) {
try {
int height = 0;
int width = 0;
double M[6] = {};
LiteMat lite_mat_rgb(input->shape()[1], input->shape()[0], input->shape()[2],
const_cast<void *>(reinterpret_cast<const void *>(input->GetBuffer())),
GetLiteCVDataType(input->type()));
LiteMat lite_mat_affine;
if (orientation == 3) {
height = lite_mat_rgb.height_;
width = lite_mat_rgb.width_;
M[0] = -1.0f;
M[1] = 0.0f;
M[2] = lite_mat_rgb.width_ - 1;
M[3] = 0.0f;
M[4] = -1.0f;
M[5] = lite_mat_rgb.height_ - 1;
} else if (orientation == 6) {
height = lite_mat_rgb.width_;
width = lite_mat_rgb.height_;
M[0] = 0.0f;
M[1] = -1.0f;
M[2] = lite_mat_rgb.height_ - 1;
M[3] = 1.0f;
M[4] = 0.0f;
M[5] = 0.0f;
} else if (orientation == 8) {
height = lite_mat_rgb.width_;
width = lite_mat_rgb.height_;
M[0] = 0.0f;
M[1] = 1.0f;
M[2] = 0.0f;
M[3] = -1.0f;
M[4] = 0.0f;
M[5] = lite_mat_rgb.width_ - 1.0f;
} else {
}
std::vector<size_t> dsize;
dsize.push_back(width);
dsize.push_back(height);
bool ret = Affine(lite_mat_rgb, lite_mat_affine, M, dsize, UINT8_C3(0, 0, 0));
CHECK_FAIL_RETURN_UNEXPECTED(ret, "Rotate failed in lite cv");
// new shape for output tensor
TensorShape new_shape = TensorShape({lite_mat_affine.height_, lite_mat_affine.width_, input->shape()[2]});
std::shared_ptr<Tensor> output_tensor;
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(new_shape, input->type(), static_cast<uchar *>(lite_mat_affine.data_ptr_),
&output_tensor));
*output = output_tensor;
} catch (std::runtime_error &e) {
RETURN_STATUS_UNEXPECTED("Error in image Rotate.");
}
return Status::OK();
}
static Status RotateAngleWithMirror(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output,
const uint64_t orientation) {
try {
int height = 0;
int width = 0;
double M[6] = {};
LiteMat lite_mat_rgb(input->shape()[1], input->shape()[0], input->shape()[2],
const_cast<void *>(reinterpret_cast<const void *>(input->GetBuffer())),
GetLiteCVDataType(input->type()));
LiteMat lite_mat_affine;
if (orientation == 2) {
height = lite_mat_rgb.height_;
width = lite_mat_rgb.width_;
M[0] = -1.0f;
M[1] = 0.0f;
M[2] = lite_mat_rgb.width_ - 1;
M[3] = 0.0f;
M[4] = 1.0f;
M[5] = 0.0f;
} else if (orientation == 5) {
height = lite_mat_rgb.width_;
width = lite_mat_rgb.height_;
M[0] = 0.0f;
M[1] = 1.0f;
M[2] = 0.0f;
M[3] = 1.0f;
M[4] = 0.0f;
M[5] = 0.0f;
} else if (orientation == 7) {
height = lite_mat_rgb.width_;
width = lite_mat_rgb.height_;
M[0] = 0.0f;
M[1] = -1.0f;
M[2] = lite_mat_rgb.height_ - 1;
M[3] = -1.0f;
M[4] = 0.0f;
M[5] = lite_mat_rgb.width_ - 1;
} else if (orientation == 4) {
height = lite_mat_rgb.height_;
width = lite_mat_rgb.width_;
M[0] = 1.0f;
M[1] = 0.0f;
M[2] = 0.0f;
M[3] = 0.0f;
M[4] = -1.0f;
M[5] = lite_mat_rgb.height_ - 1;
} else {
}
std::vector<size_t> dsize;
dsize.push_back(width);
dsize.push_back(height);
bool ret = Affine(lite_mat_rgb, lite_mat_affine, M, dsize, UINT8_C3(0, 0, 0));
CHECK_FAIL_RETURN_UNEXPECTED(ret, "Rotate failed in lite cv");
// new shape for output tensor
TensorShape new_shape = TensorShape({lite_mat_affine.height_, lite_mat_affine.width_, input->shape()[2]});
std::shared_ptr<Tensor> output_tensor;
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(new_shape, input->type(), static_cast<uchar *>(lite_mat_affine.data_ptr_),
&output_tensor));
*output = output_tensor;
} catch (std::runtime_error &e) {
RETURN_STATUS_UNEXPECTED("Error in image Rotate.");
}
return Status::OK();
}
static bool IsMirror(int orientation) {
if (orientation == 2 || orientation == 4 || orientation == 5 || orientation == 7) {
return true;
}
return false;
}
// rotate the image by EXIF orientation
Status Rotate(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, const uint64_t orientation) {
if (input->Rank() != 3) {
RETURN_STATUS_UNEXPECTED("Input Tensor is not in shape of <H,W,C>");
}
if (input->type() != DataType::DE_FLOAT32 && input->type() != DataType::DE_UINT8) {
RETURN_STATUS_UNEXPECTED("Only float32, uint8 support in Pad");
}
if (!IsMirror(orientation)) {
return RotateAngleWithOutMirror(input, output, orientation);
} else {
return RotateAngleWithMirror(input, output, orientation);
}
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/kernels/image/lite_image_utils.h
View File

@ -109,6 +109,8 @@ Status Resize(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *out
Status Pad(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, const int32_t &pad_top,
const int32_t &pad_bottom, const int32_t &pad_left, const int32_t &pad_right, const BorderType &border_types,
uint8_t fill_r = 0, uint8_t fill_g = 0, uint8_t fill_b = 0);
Status Rotate(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, const uint64_t orientation);
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_

8
mindspore/ccsrc/minddata/dataset/util/status.cc
View File

@ -16,7 +16,12 @@
#include "minddata/dataset/util/status.h"
#include <sstream>
#include "utils/ms_utils.h"
#ifndef ENABLE_ANDROID
#include "minddata/dataset/util/task_manager.h"
#else
#include "minddata/dataset/util/log_adapter.h"
#endif
namespace mindspore {
namespace dataset {
@ -104,11 +109,14 @@ Status::Status(const StatusCode code, const std::string &msg) : code_(code), err
Status::Status(const StatusCode code, int line_of_code, const char *file_name, const std::string &extra) {
code_ = code;
std::ostringstream ss;
#ifndef ENABLE_ANDROID
ss << "Thread ID " << this_thread::get_id() << " " << CodeAsString(code) << ". ";
if (!extra.empty()) {
ss << extra;
}
ss << "\n";
#endif
ss << "Line of code : " << line_of_code << "\n";
if (file_name != nullptr) {
ss << "File : " << file_name << "\n";

9
mindspore/lite/minddata/CMakeLists.txt
View File

@ -81,12 +81,12 @@ AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/util MINDDATA_UTIL_SRC_FILES)
AUX_SOURCE_DIRECTORY(${MINDDATA_DIR}/kernels/image/lite_cv MINDDATA_KERNELS_IMAGE_LITE_CV_FILES)
if (PLATFORM_ARM32 OR PLATFORM_ARM64)
if (BUILD_MINDDATA STREQUAL "full")
set(BUILD_MINDDATA "wrapper")
endif ()
if (BUILD_MINDDATA STREQUAL "full")
set(BUILD_MINDDATA "wrapper")
endif ()
if (BUILD_MINDDATA STREQUAL "full")
include_directories("${CMAKE_SOURCE_DIR}/../ccsrc/minddata/dataset/kernels/image")
list(REMOVE_ITEM MINDDATA_API_SRC_FILES
@ -334,6 +334,7 @@ elseif (BUILD_MINDDATA STREQUAL "wrapper")
${MINDDATA_DIR}/kernels/data/random_choice_op.cc
${MINDDATA_DIR}/kernels/data/type_cast_op.cc
${MINDDATA_DIR}/kernels/data/data_utils.cc
${MINDDATA_DIR}/kernels/image/exif_utils.cc
${CMAKE_CURRENT_SOURCE_DIR}/wrapper/MDToDApi.cc
${CMAKE_CURRENT_SOURCE_DIR}/wrapper/album_op_android.cc
)

2
mindspore/lite/minddata/wrapper/MDToDApi.h
View File

@ -17,6 +17,8 @@
#define DATASET_MDTODAPI_H_
#include <stdint.h>
#include <sys/types.h>
class MDToDApi;
typedef struct MDToDBuff {

65
mindspore/lite/minddata/wrapper/album_op_android.cc
View File

@ -18,6 +18,7 @@
#include <iomanip>
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/kernels/image/lite_image_utils.h"
#include "minddata/dataset/kernels/image/exif_utils.h"
namespace mindspore {
namespace dataset {
@ -33,7 +34,8 @@ AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string
current_cnt_(0),
dirname_offset_(0),
sampler_(false),
sampler_index_(0) {
sampler_index_(0),
rotate_(true) {
PrescanEntry();
}
@ -48,7 +50,8 @@ AlbumOp::AlbumOp(const std::string &file_dir, bool do_decode, const std::string
current_cnt_(0),
dirname_offset_(0),
sampler_(true),
sampler_index_(index) {
sampler_index_(index),
rotate_(true) {
PrescanEntry();
}
@ -168,6 +171,7 @@ bool AlbumOp::CheckImageType(const std::string &file_name, bool *valid) {
Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col_num, TensorPtr *tensor) {
TensorPtr image;
TensorPtr rotate_tensor;
std::ifstream fs;
fs.open(image_file_path, std::ios::binary | std::ios::in);
if (fs.fail()) {
@ -204,21 +208,66 @@ Status AlbumOp::LoadImageTensor(const std::string &image_file_path, uint32_t col
}
// if it is a jpeg image, load and try to decode
RETURN_IF_NOT_OK(Tensor::CreateFromFile(image_file_path, &image));
Status rc;
if (decode_ && valid) {
Status rc = Decode(image, tensor);
if (rc.IsError()) {
RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
return Status::OK();
int orientation = GetOrientation(image_file_path);
if (orientation > 1 && this->rotate_) {
rc = Decode(image, &rotate_tensor);
if (rc.IsError()) {
RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
return Status::OK();
}
rc = Rotate(rotate_tensor, tensor, orientation);
if (rc.IsError()) {
RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
return Status::OK();
}
} else {
rc = Decode(image, tensor);
if (rc.IsError()) {
RETURN_IF_NOT_OK(LoadEmptyTensor(col_num, tensor));
return Status::OK();
}
}
}
// row->push_back(std::move(image));
return Status::OK();
}
// get orientation from EXIF file
int AlbumOp::GetOrientation(const std::string &folder_path) {
FILE *fp = fopen(folder_path.c_str(), "rb");
if (!fp) {
MS_LOG(WARNING) << "Can't read file for EXIF: file = " << folder_path;
return 0;
}
fseek(fp, 0, SEEK_END);
int64_t fsize = ftell(fp);
rewind(fp);
unsigned char *buf = new unsigned char[fsize];
if (fread(buf, 1, fsize, fp) != fsize) {
MS_LOG(WARNING) << "read file size error for EXIF: file = " << folder_path;
delete[] buf;
fclose(fp);
return 0;
}
fclose(fp);
// Parse EXIF
mindspore::dataset::ExifInfo result;
int code = result.parseOrientation(buf, fsize);
delete[] buf;
if (code == 0) {
MS_LOG(WARNING) << "Error parsing EXIF, use default code = " << code << ".";
}
return code;
}
Status AlbumOp::LoadStringArrayTensor(const nlohmann::json &json_obj, uint32_t col_num, TensorPtr *tensor) {
std::vector<std::string> data = json_obj.get<std::vector<std::string>>();
MS_LOG(WARNING) << "String array label found: " << data << ".";
MS_LOG(INFO) << "String array label found: " << data << ".";
// TensorPtr label;
RETURN_IF_NOT_OK(Tensor::CreateFromVector(data, tensor));
// row->push_back(std::move(label));

12
mindspore/lite/minddata/wrapper/album_op_android.h
View File

@ -30,7 +30,6 @@
#include "minddata/dataset/engine/data_buffer.h"
#include "minddata/dataset/engine/data_schema.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/queue.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
@ -50,6 +49,7 @@ class AlbumOp {
/// \param[in] do_decode - decode image files
/// \param[in] schema_file - schema file
/// \param[in] exts - set of file extensions to read, if empty, read everything under the dir
/// \param[in] rotate - rotate image exif orientation
AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
const std::set<std::string> &exts);
@ -59,6 +59,7 @@ class AlbumOp {
/// \param[in] schema_file - schema file
/// \param[in] exts - set of file extensions to read, if empty, read everything under the dir
/// \param[in] index - the specific file index
/// \param[in] rotate - rotate image exif orientation
AlbumOp(const std::string &file_dir, bool do_decode, const std::string &schema_file,
const std::set<std::string> &exts, uint32_t index);
@ -82,6 +83,10 @@ class AlbumOp {
// @return Name of the current Op
std::string Name() const { return "AlbumOp"; }
// Op name DisableRotate
// @return
void DisableRotate() { this->rotate_ = false; }
private:
/// \brief Load image to tensor
/// \param[in] image_file Image name of file
@ -153,6 +158,10 @@ class AlbumOp {
Status LoadTensorRow(row_id_type row_id, const std::string &file,
std::unordered_map<std::string, std::shared_ptr<Tensor>> *map_row);
/// \brief get image exif orientation
/// \param[in] file file path
int GetOrientation(const std::string &file);
std::string folder_path_; // directory of image folder
bool decode_;
std::vector<std::string> columns_to_load_;
@ -167,6 +176,7 @@ class AlbumOp {
int64_t sampler_index_;
std::vector<std::string> image_rows_;
std::unordered_map<std::string, int32_t> column_name_id_map_;
bool rotate_;
};
} // namespace dataset
} // namespace mindspore