/* * ESPRESSIF MIT License * * Copyright (c) 2018 * * Permission is hereby granted for use on ESPRESSIF SYSTEMS products only, in which case, * it is free of charge, to any person obtaining a copy of this software and associated * documentation files (the "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the Software is furnished * to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all copies or * substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * */ #pragma once #ifdef __cplusplus extern "C" { #endif #include #include #include "mtmn.h" #define LANDMARKS_NUM (10) #define MAX_VALID_COUNT_PER_IMAGE (30) #define DL_IMAGE_MIN(A, B) ((A) < (B) ? (A) : (B)) #define DL_IMAGE_MAX(A, B) ((A) < (B) ? (B) : (A)) #define RGB565_MASK_RED 0xF800 #define RGB565_MASK_GREEN 0x07E0 #define RGB565_MASK_BLUE 0x001F typedef enum { BINARY, /*!< binary */ } en_threshold_mode; typedef struct { fptp_t landmark_p[LANDMARKS_NUM]; /*!< landmark struct */ } landmark_t; typedef struct { fptp_t box_p[4]; /*!< box struct */ } box_t; typedef struct tag_box_list { uint8_t *category; /*!< The category of the corresponding box */ fptp_t *score; /*!< The confidence score of the class corresponding to the box */ box_t *box; /*!< Anchor boxes or predicted boxes*/ landmark_t *landmark; /*!< The landmarks corresponding to the box */ int len; /*!< The num of the boxes */ } box_array_t; typedef struct tag_image_box { struct tag_image_box *next; /*!< Next image_box_t */ uint8_t category; fptp_t score; /*!< The confidence score of the class corresponding to the box */ box_t box; /*!< Anchor boxes or predicted boxes */ box_t offset; /*!< The predicted anchor-based offset */ landmark_t landmark; /*!< The landmarks corresponding to the box */ } image_box_t; typedef struct tag_image_list { image_box_t *head; /*!< The current head of the image_list */ image_box_t *origin_head; /*!< The original head of the image_list */ int len; /*!< Length of the image_list */ } image_list_t; /** * @brief Get the width and height of the box. * * @param box Input box * @param w Resulting width of the box * @param h Resulting height of the box */ static inline void image_get_width_and_height(box_t *box, float *w, float *h) { *w = box->box_p[2] - box->box_p[0] + 1; *h = box->box_p[3] - box->box_p[1] + 1; } /** * @brief Get the area of the box. * * @param box Input box * @param area Resulting area of the box */ static inline void image_get_area(box_t *box, float *area) { float w, h; image_get_width_and_height(box, &w, &h); *area = w * h; } /** * @brief calibrate the boxes by offset * * @param image_list Input boxes * @param image_height Height of the original image * @param image_width Width of the original image */ static inline void image_calibrate_by_offset(image_list_t *image_list, int image_height, int image_width) { for (image_box_t *head = image_list->head; head; head = head->next) { float w, h; image_get_width_and_height(&(head->box), &w, &h); head->box.box_p[0] = DL_IMAGE_MAX(0, head->box.box_p[0] + head->offset.box_p[0] * w); head->box.box_p[1] = DL_IMAGE_MAX(0, head->box.box_p[1] + head->offset.box_p[1] * w); head->box.box_p[2] += head->offset.box_p[2] * w; if (head->box.box_p[2] > image_width) { head->box.box_p[2] = image_width - 1; head->box.box_p[0] = image_width - w; } head->box.box_p[3] += head->offset.box_p[3] * h; if (head->box.box_p[3] > image_height) { head->box.box_p[3] = image_height - 1; head->box.box_p[1] = image_height - h; } } } /** * @brief calibrate the landmarks * * @param image_list Input landmarks */ static inline void image_landmark_calibrate(image_list_t *image_list) { for (image_box_t *head = image_list->head; head; head = head->next) { float w, h; image_get_width_and_height(&(head->box), &w, &h); head->landmark.landmark_p[0] = head->box.box_p[0] + head->landmark.landmark_p[0] * w; head->landmark.landmark_p[1] = head->box.box_p[1] + head->landmark.landmark_p[1] * h; head->landmark.landmark_p[2] = head->box.box_p[0] + head->landmark.landmark_p[2] * w; head->landmark.landmark_p[3] = head->box.box_p[1] + head->landmark.landmark_p[3] * h; head->landmark.landmark_p[4] = head->box.box_p[0] + head->landmark.landmark_p[4] * w; head->landmark.landmark_p[5] = head->box.box_p[1] + head->landmark.landmark_p[5] * h; head->landmark.landmark_p[6] = head->box.box_p[0] + head->landmark.landmark_p[6] * w; head->landmark.landmark_p[7] = head->box.box_p[1] + head->landmark.landmark_p[7] * h; head->landmark.landmark_p[8] = head->box.box_p[0] + head->landmark.landmark_p[8] * w; head->landmark.landmark_p[9] = head->box.box_p[1] + head->landmark.landmark_p[9] * h; } } /** * @brief Convert a rectangular box into a square box * * @param boxes Input box * @param width Width of the orignal image * @param height height of the orignal image */ static inline void image_rect2sqr(box_array_t *boxes, int width, int height) { for (int i = 0; i < boxes->len; i++) { box_t *box = &(boxes->box[i]); int x1 = round(box->box_p[0]); int y1 = round(box->box_p[1]); int x2 = round(box->box_p[2]); int y2 = round(box->box_p[3]); int w = x2 - x1 + 1; int h = y2 - y1 + 1; int l = DL_IMAGE_MAX(w, h); box->box_p[0] = DL_IMAGE_MAX(round(DL_IMAGE_MAX(0, x1) + 0.5 * (w - l)), 0); box->box_p[1] = DL_IMAGE_MAX(round(DL_IMAGE_MAX(0, y1) + 0.5 * (h - l)), 0); box->box_p[2] = box->box_p[0] + l - 1; if (box->box_p[2] > width) { box->box_p[2] = width - 1; box->box_p[0] = width - l; } box->box_p[3] = box->box_p[1] + l - 1; if (box->box_p[3] > height) { box->box_p[3] = height - 1; box->box_p[1] = height - l; } } } /**@{*/ /** * @brief Convert RGB565 image to RGB888 image * * @param in Input RGB565 image * @param dst Resulting RGB888 image */ static inline void rgb565_to_888(uint16_t in, uint8_t *dst) { /*{{{*/ in = (in & 0xFF) << 8 | (in & 0xFF00) >> 8; dst[2] = (in & RGB565_MASK_BLUE) << 3; // blue dst[1] = (in & RGB565_MASK_GREEN) >> 3; // green dst[0] = (in & RGB565_MASK_RED) >> 8; // red // dst[0] = (in & 0x1F00) >> 5; // dst[1] = ((in & 0x7) << 5) | ((in & 0xE000) >> 11); // dst[2] = in & 0xF8; } /*}}}*/ static inline void rgb565_to_888_q16(uint16_t in, int16_t *dst) { /*{{{*/ in = (in & 0xFF) << 8 | (in & 0xFF00) >> 8; dst[2] = (in & RGB565_MASK_BLUE) << 3; // blue dst[1] = (in & RGB565_MASK_GREEN) >> 3; // green dst[0] = (in & RGB565_MASK_RED) >> 8; // red // dst[0] = (in & 0x1F00) >> 5; // dst[1] = ((in & 0x7) << 5) | ((in & 0xE000) >> 11); // dst[2] = in & 0xF8; } /*}}}*/ /**@}*/ /** * @brief Convert RGB888 image to RGB565 image * * @param in Resulting RGB565 image * @param r The red channel of the Input RGB888 image * @param g The green channel of the Input RGB888 image * @param b The blue channel of the Input RGB888 image */ static inline void rgb888_to_565(uint16_t *in, uint8_t r, uint8_t g, uint8_t b) { /*{{{*/ uint16_t rgb565 = 0; rgb565 = ((r >> 3) << 11); rgb565 |= ((g >> 2) << 5); rgb565 |= (b >> 3); rgb565 = (rgb565 & 0xFF) << 8 | (rgb565 & 0xFF00) >> 8; *in = rgb565; } /*}}}*/ /** * @brief Filter out the resulting boxes whose confidence score is lower than the threshold and convert the boxes to the actual boxes on the original image.((x, y, w, h) -> (x1, y1, x2, y2)) * * @param score Confidence score of the boxes * @param offset The predicted anchor-based offset * @param landmark The landmarks corresponding to the box * @param width Height of the original image * @param height Width of the original image * @param anchor_number Anchor number of the detection output feature map * @param anchors_size The anchor size * @param score_threshold Threshold of the confidence score * @param stride * @param resized_height_scale * @param resized_width_scale * @param do_regression * @return image_list_t* */ image_list_t *image_get_valid_boxes(fptp_t *score, fptp_t *offset, fptp_t *landmark, int width, int height, int anchor_number, int *anchors_size, fptp_t score_threshold, int stride, fptp_t resized_height_scale, fptp_t resized_width_scale, bool do_regression); /** * @brief Sort the resulting box lists by their confidence score. * * @param image_sorted_list The sorted box list. * @param insert_list The box list that have not been sorted. */ void image_sort_insert_by_score(image_list_t *image_sorted_list, const image_list_t *insert_list); /** * @brief Run NMS algorithm * * @param image_list The input boxes list * @param nms_threshold NMS threshold * @param same_area The flag of boxes with same area */ void image_nms_process(image_list_t *image_list, fptp_t nms_threshold, int same_area); /** * @brief Resize an image to half size * * @param dimage The output image * @param dw Width of the output image * @param dh Height of the output image * @param dc Channel of the output image * @param simage Source image * @param sw Width of the source image * @param sc Channel of the source image */ void image_zoom_in_twice(uint8_t *dimage, int dw, int dh, int dc, uint8_t *simage, int sw, int sc); /** * @brief Resize the image in RGB888 format via bilinear interpolation * * @param dst_image The output image * @param src_image Source image * @param dst_w Width of the output image * @param dst_h Height of the output image * @param dst_c Channel of the output image * @param src_w Width of the source image * @param src_h Height of the source image */ void image_resize_linear(uint8_t *dst_image, uint8_t *src_image, int dst_w, int dst_h, int dst_c, int src_w, int src_h); /** * @brief Crop, rotate and zoom the image in RGB888 format, * * @param corp_image The output image * @param src_image Source image * @param rotate_angle Rotate angle * @param ratio scaling ratio * @param center Center of rotation */ void image_cropper(uint8_t *corp_image, uint8_t *src_image, int dst_w, int dst_h, int dst_c, int src_w, int src_h, float rotate_angle, float ratio, float *center); /** * @brief Convert the rgb565 image to the rgb888 image * * @param m The output rgb888 image * @param bmp The input rgb565 image * @param count Total pixels of the rgb565 image */ void image_rgb565_to_888(uint8_t *m, uint16_t *bmp, int count); /** * @brief Convert the rgb888 image to the rgb565 image * * @param bmp The output rgb565 image * @param m The input rgb888 image * @param count Total pixels of the rgb565 image */ void image_rgb888_to_565(uint16_t *bmp, uint8_t *m, int count); /** * @brief draw rectangle on the rgb565 image * * @param buf Input image * @param boxes Rectangle Boxes * @param width Width of the input image */ void draw_rectangle_rgb565(uint16_t *buf, box_array_t *boxes, int width); /** * @brief draw rectangle on the rgb888 image * * @param buf Input image * @param boxes Rectangle Boxes * @param width Width of the input image */ void draw_rectangle_rgb888(uint8_t *buf, box_array_t *boxes, int width); /** * @brief Get the pixel difference of two images * * @param dst The output pixel difference * @param src1 Input image 1 * @param src2 Input image 2 * @param count Total pixels of the input image */ void image_abs_diff(uint8_t *dst, uint8_t *src1, uint8_t *src2, int count); /** * @brief Binarize an image to 0 and value. * * @param dst The output image * @param src Source image * @param threshold Threshold of binarization * @param value The value of binarization * @param count Total pixels of the input image * @param mode Threshold mode */ void image_threshold(uint8_t *dst, uint8_t *src, int threshold, int value, int count, en_threshold_mode mode); /** * @brief Erode the image * * @param dst The output image * @param src Source image * @param src_w Width of the source image * @param src_h Height of the source image * @param src_c Channel of the source image */ void image_erode(uint8_t *dst, uint8_t *src, int src_w, int src_h, int src_c); typedef float matrixType; typedef struct { int w; /*!< width */ int h; /*!< height */ matrixType **array; /*!< array */ } Matrix; /** * @brief Allocate a 2d matrix * * @param h Height of matrix * @param w Width of matrix * @return Matrix* 2d matrix */ Matrix *matrix_alloc(int h, int w); /** * @brief Free a 2d matrix * * @param m 2d matrix */ void matrix_free(Matrix *m); /** * @brief Get the similarity matrix of similarity transformation * * @param srcx Source x coordinates * @param srcy Source y coordinates * @param dstx Destination x coordinates * @param dsty Destination y coordinates * @param num The number of the coordinates * @return Matrix* The resulting transformation matrix */ Matrix *get_similarity_matrix(float *srcx, float *srcy, float *dstx, float *dsty, int num); /** * @brief Get the affine transformation matrix * * @param srcx Source x coordinates * @param srcy Source y coordinates * @param dstx Destination x coordinates * @param dsty Destination y coordinates * @return Matrix* The resulting transformation matrix */ Matrix *get_affine_transform(float *srcx, float *srcy, float *dstx, float *dsty); /** * @brief Applies an affine transformation to an image * * @param img Input image * @param crop Dst output image that has the size dsize and the same type as src * @param M Affine transformation matrix */ void warp_affine(dl_matrix3du_t *img, dl_matrix3du_t *crop, Matrix *M); /** * @brief Resize the image in RGB888 format via bilinear interpolation, and quantify the output image * * @param dst_image Quantized output image * @param src_image Input image * @param dst_w Width of the output image * @param dst_h Height of the output image * @param dst_c Channel of the output image * @param src_w Width of the input image * @param src_h Height of the input image * @param shift Shift parameter of quantization. */ void image_resize_linear_q(qtp_t *dst_image, uint8_t *src_image, int dst_w, int dst_h, int dst_c, int src_w, int src_h, int shift); /** * @brief Preprocess the input image of object detection model. The process is like this: resize -> normalize -> quantify * * @param image Input image, RGB888 format. * @param input_w Width of the input image. * @param input_h Height of the input image. * @param target_size Target size of the model input image. * @param exponent Exponent of the quantized model input image. * @param process_mode Process mode. 0: resize with padding to keep height == width. 1: resize without padding, height != width. * @return dl_matrix3dq_t* The resulting preprocessed image. */ dl_matrix3dq_t *image_resize_normalize_quantize(uint8_t *image, int input_w, int input_h, int target_size, int exponent, int process_mode); /** * @brief Resize the image in RGB565 format via mean neighbour interpolation, and quantify the output image * * @param dimage Quantized output image. * @param simage Input image. * @param dw Width of the allocated output image memory. * @param dc Channel of the allocated output image memory. * @param sw Width of the input image. * @param sh Height of the input image. * @param tw Target width of the output image. * @param th Target height of the output image. * @param shift Shift parameter of quantization. */ void image_resize_shift_fast(qtp_t *dimage, uint16_t *simage, int dw, int dc, int sw, int sh, int tw, int th, int shift); /** * @brief Resize the image in RGB565 format via nearest neighbour interpolation, and quantify the output image * * @param dimage Quantized output image. * @param simage Input image. * @param dw Width of the allocated output image memory. * @param dc Channel of the allocated output image memory. * @param sw Width of the input image. * @param sh Height of the input image. * @param tw Target width of the output image. * @param th Target height of the output image. * @param shift Shift parameter of quantization. */ void image_resize_nearest_shift(qtp_t *dimage, uint16_t *simage, int dw, int dc, int sw, int sh, int tw, int th, int shift); /** * @brief Crop the image in RGB565 format and resize it to target size, then quantify the output image * * @param dimage Quantized output image. * @param simage Input image. * @param dw Target size of the output image. * @param sw Width of the input image. * @param sh Height of the input image. * @param x1 The x coordinate of the upper left corner of the cropped area * @param y1 The y coordinate of the upper left corner of the cropped area * @param x2 The x coordinate of the lower right corner of the cropped area * @param y2 The y coordinate of the lower right corner of the cropped area * @param shift Shift parameter of quantization. */ void image_crop_shift_fast(qtp_t *dimage, uint16_t *simage, int dw, int sw, int sh, int x1, int y1, int x2, int y2, int shift); #ifdef __cplusplus } #endif