Mirror/AI-on-the-edge-device
1
0
Fork 0
mirror of https://github.com/jomjol/AI-on-the-edge-device.git synced 2025-12-09 04:56:53 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rolling 20220716_2

Browse Source
This commit is contained in:
jomjol
2022-07-16 20:59:09 +02:00
parent 0b039e8d8c
commit 17fd0f96df
166 changed files with 10471 additions and 1691 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
code/components/esp-nn/src/common/common_functions.h
View File

@@ -41,15 +41,39 @@
__NN_FORCE_INLINE__ int32_t esp_nn_clz32(uint32_t in)
{
#if CONFIG_IDF_TARGET_ARCH_XTENSA
__asm__ volatile("nsau %0, %0" : "+r" (in));
return in;
}
__NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64)
{
int32_t sign = (int32_t) (val64 >> 63);
int32_t to_add = sign & ((1ul << 31) - 1);
return (int32_t) ((int64_t) (val64 + to_add) >> 31);
#elif defined(__GNUC__)
return __builtin_clz(in);
#else
int32_t count = 32;
uint32_t x = in, y = in >> 16;
if (y != 0) {
count -= 16;
x = y;
}
y = x >> 8;
if (y != 0) {
count -= 8;
x = y;
}
y = x >> 4;
if (y != 0) {
count -= 4;
x = y;
}
y = x >> 2;
if (y != 0) {
count -= 2;
x = y;
}
y = x >> 1;
if (y != 0) {
return count - 2;
}
return count - x;
#endif
}
/**
@@ -57,8 +81,19 @@ __NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64)
*/
__NN_FORCE_INLINE__ int32_t esp_nn_saturate8(int32_t in)
{
#if CONFIG_IDF_TARGET_ARCH_XTENSA
__asm__ volatile("clamps %0, %0, 7" : "+a"(in));
return in;
#else
return max(INT8_MIN, min(in, INT8_MAX));
#endif
}
__NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64)
{
int32_t sign = (int32_t) (val64 >> 63);
int32_t to_add = sign & ((1ul << 31) - 1);
return (int32_t) ((int64_t) (val64 + to_add) >> 31);
}
__NN_FORCE_INLINE__ int32_t esp_nn_sat_round_doubling_high_mul(int32_t in0, int32_t in1)
@@ -144,7 +179,7 @@ static void esp_nn_aligned_s8_pad_with_value(const int8_t *src, int8_t *dst,
const uint16_t pad_ht)
{
/* memset with pad_val */
memset(dst, pad_val, ((input_wd + 2 * pad_wd) * (input_ht + 2 * pad_ht)) * channels * 2);
memset(dst, pad_val, ((input_wd + 2 * pad_wd) * (input_ht + 2 * pad_ht)) * channels);
dst += (pad_wd + input_wd + pad_wd) * channels;
for (int i = 0; i < input_ht; i++) {
@@ -156,7 +191,6 @@ static void esp_nn_aligned_s8_pad_with_value(const int8_t *src, int8_t *dst,
}
}
#if 0
static void esp_nn_aligned_s8_pad_end_with_value(const int8_t *src, int8_t *dst,
const uint16_t input_wd,
const uint16_t input_ht,
@@ -169,13 +203,16 @@ static void esp_nn_aligned_s8_pad_end_with_value(const int8_t *src, int8_t *dst,
for (int j = 0; j < input_wd * channels; j++) {
*dst++ = *src++;
}
memset(dst, pad_val, pad_wd * channels);
dst += pad_wd * channels;
if (pad_wd) {
memset(dst, pad_val, pad_wd * channels);
dst += pad_wd * channels;
}
}
/* pad end `pad_ht` lines at end */
memset(dst, pad_val, (input_wd + pad_wd) * pad_ht * channels);
if (pad_ht) {
memset(dst, pad_val, (input_wd + pad_wd) * pad_ht * channels);
}
}
#endif
/**
* @brief convert 8 bit input data to 16 bit

56
code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c
View File

@@ -12,16 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <esp_nn_defs.h>
#include <common_functions.h>
int esp_nn_get_conv_scratch_size_ansi(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_ch,
const uint16_t out_ch,
const uint16_t filter_wd,
const uint16_t filter_ht)
int esp_nn_get_conv_scratch_size_ansi(const data_dims_t *input_dims,
const data_dims_t *filter_dims,
const data_dims_t *output_dims,
const conv_params_t *conv_params)
{
return 0;
}
@@ -108,29 +106,35 @@ void esp_nn_conv_u8_ansi(const uint8_t *input_data,
* Assumption 2: Pointers are valid
* Assumption 3: dialation width = 1
*/
void esp_nn_conv_s8_ansi(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
void esp_nn_conv_s8_ansi(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
const conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t in_channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const uint16_t out_channels = output_dims->channels;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {

185
code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c
View File

@@ -12,30 +12,30 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <esp_nn_defs.h>
#include <common_functions.h>
static int16_t *scratch_buffer = NULL;
extern void esp_nn_conv_s16_mult8_1x1_esp32s3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const int16_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max,
void *buffer /* scratch buffer */);
extern void esp_nn_conv_s8_mult8_1x1_esp32s3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const int8_t *filter_aligned,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max,
void *buffer /* scratch buffer */);
extern void esp_nn_conv_s16_mult4_1x1_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
@@ -81,34 +81,40 @@ extern void esp_nn_aligned_s8_to_s16_with_offset_esp32s3(const int8_t *src, int1
extern void esp_nn_s8_to_s16_esp32s3(const int8_t *src, int16_t *dst, const int size);
static void esp_nn_conv_s8_unrolled(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
static void esp_nn_conv_s8_unrolled(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
const conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t in_ch = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const uint16_t out_ch = output_dims->channels;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
for (out_ch_idx = 0; out_ch_idx < out_ch; out_ch_idx++) {
int32_t conv_out = 0;
const int32_t base_y = stride_ht * out_y - pad_ht;
@@ -124,10 +130,10 @@ static void esp_nn_conv_s8_unrolled(const int8_t *input_data,
for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t in_row = base_y + filter_y_idx;
const int32_t in_col = base_x + filter_x_idx;
int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd +
(filter_y_idx * filter_wd + filter_x_idx) * in_channels;
for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
int32_t input_base_offset = (in_row * input_wd + in_col) * in_ch;
int32_t filter_base_offset = out_ch_idx * in_ch * filter_ht * filter_wd +
(filter_y_idx * filter_wd + filter_x_idx) * in_ch;
for (in_ch_idx = 0; in_ch_idx < in_ch; in_ch_idx++) {
conv_out +=
(input_data[input_base_offset + in_ch_idx] + input_offset) *
filter_data[filter_base_offset + in_ch_idx];
@@ -332,18 +338,35 @@ static void esp_nn_conv_s8_pad_valid_ch3_3x3(const int8_t *input_data,
}
}
int esp_nn_get_conv_scratch_size_esp32s3(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_ch,
const uint16_t out_ch,
const uint16_t filter_wd,
const uint16_t filter_ht)
int esp_nn_get_conv_scratch_size_esp32s3(const data_dims_t *input_dims,
const data_dims_t *filter_dims,
const data_dims_t *output_dims,
const conv_params_t *conv_params)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t in_ch = input_dims->channels;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_ch = output_dims->channels;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
int filter_size = filter_wd * filter_ht * in_ch * out_ch;
int input_size = input_wd * input_ht * in_ch;
int transpose_buf_size = 8 * in_ch; /* to store intermediate data */
int transpose_buf_size = 2 * (8 * in_ch); /* to store intermediate data */
if (input_wd * input_ht < 8) {
transpose_buf_size = 0; // not using this for leftover
}
int align_buf_size = 32; /* extra buffer for alignment */
return 2 * (filter_size + input_size + transpose_buf_size) + align_buf_size;
if (in_ch % 8 == 0 && filter_wd == 1 && filter_ht == 1 &&
pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
return filter_size + transpose_buf_size + align_buf_size;
}
return 2 * (filter_size + input_size) + transpose_buf_size + align_buf_size;
}
void esp_nn_set_conv_scratch_buf_esp32s3(void *buf)
@@ -351,29 +374,35 @@ void esp_nn_set_conv_scratch_buf_esp32s3(void *buf)
scratch_buffer = (int16_t *) buf;
}
void esp_nn_conv_s8_esp32s3(const int8_t *input,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
void esp_nn_conv_s8_esp32s3(const data_dims_t *input_dims,
const int8_t *input,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
const conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const uint16_t out_channels = output_dims->channels;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
int filter_size = filter_wd * filter_ht * channels * out_channels;
int input_size = input_wd * input_ht * channels;
int align_len = 16 - (filter_size & 15);
@@ -387,15 +416,16 @@ void esp_nn_conv_s8_esp32s3(const int8_t *input,
if (channels % 8 == 0 && filter_wd == 1 && filter_ht == 1 &&
pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
int scratch_offset = (int) (filter_data16 + filter_size);
int8_t *filter_aligned = (int8_t *) scratch_buffer;
int scratch_offset = (int) (filter_aligned + filter_size);
void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15));
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_conv_s16_mult8_1x1_esp32s3(
input, input_wd, input_ht, channels, input_offset, filter_data16,
memcpy(filter_aligned, filter_data, filter_size); // copy to aligned address
esp_nn_conv_s8_mult8_1x1_esp32s3(
input, input_wd, input_ht, channels, input_offset, filter_aligned,
bias, out_data, out_wd, out_ht, out_channels, out_offset,
out_shift, out_mult, activation_min, activation_max, scratch_buf);
} else if (channels % 4 == 0 && filter_wd == 1 && filter_ht == 1 &&
(input_wd * input_ht) % 16 == 0 && /* TODO: remove this check */
(input_wd * input_ht) % 4 == 0 && /* TODO: remove this check */
pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
int scratch_offset = (int) (input_data16 + input_size);
void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15));
@@ -427,10 +457,7 @@ void esp_nn_conv_s8_esp32s3(const int8_t *input,
}
} else {
/* Basic unrolled version */
esp_nn_conv_s8_unrolled(input, input_wd, input_ht, channels, input_offset,
pad_wd, pad_ht, stride_wd, stride_ht,
filter_data, filter_wd, filter_ht, bias,
out_data, out_wd, out_ht, out_channels, out_offset, out_shift,
out_mult, activation_min, activation_max);
esp_nn_conv_s8_unrolled(input_dims, input, filter_dims, filter_data,
bias, output_dims, out_data, conv_params, quant_data);
}
}

179
code/components/esp-nn/src/convolution/esp_nn_conv_opt.c Normal file
View File

@@ -0,0 +1,179 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <esp_nn_defs.h>
#include <common_functions.h>
int esp_nn_get_conv_scratch_size_opt(const data_dims_t *input_dims,
const data_dims_t *filter_dims,
const data_dims_t *output_dims,
const conv_params_t *conv_params)
{
return 0;
}
void esp_nn_set_conv_scratch_buf_opt(const void *buf)
{
}
__attribute__ ((noinline))
static void esp_nn_conv_s8_1x1(const data_dims_t *input_dims,
const int8_t *input_data,
const int8_t *filter_data,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t in_channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const uint16_t out_channels = output_dims->channels;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
for (int32_t in_row = 0; in_row < out_ht * stride_ht; in_row += stride_ht) {
for (int32_t in_col = 0; in_col < out_wd * stride_wd; in_col += stride_wd) {
const int32_t *out_mult = quant_data->mult;
const int32_t *out_shift = quant_data->shift;
const int8_t *filter_ptr = filter_data;
const int8_t *input_base_ptr = input_data + (in_row * input_wd + in_col) * in_channels;
int32_t out_ch_idx = 0;
for (; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
const int8_t *input_ptr = input_base_ptr;
int32_t in_ch_idx = 0;
for (; in_ch_idx < in_channels - 3; in_ch_idx += 4) {
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
}
for (; in_ch_idx < in_channels; in_ch_idx ++) {
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
}
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, *out_mult++, *out_shift++);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}
/**
* Assumption 1: i/p channels == o/p channels
* Assumption 2: Pointers are valid
* Assumption 3: dialation width = 1
*/
void esp_nn_conv_s8_opt(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
if (filter_wd == 1 && filter_ht == 1) {
esp_nn_conv_s8_1x1(input_dims, input_data, filter_data, bias,
output_dims, out_data, conv_params, quant_data);
return;
}
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t in_channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const uint16_t out_channels = output_dims->channels;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
int32_t out_ch_idx, out_y, out_x, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
const int32_t base_y = stride_ht * out_y - pad_ht;
const int32_t base_x = stride_wd * out_x - pad_wd;
const int32_t filter_y_start = max(0, -base_y);
const int32_t filter_x_start = max(0, -base_x);
const int32_t filter_y_end = min(filter_ht, input_ht - base_y);
const int32_t filter_x_end = min(filter_wd, input_wd - base_x);
for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t in_row = base_y + filter_y_idx;
const int32_t in_col = base_x + filter_x_idx;
const int8_t *input_ptr = input_data +
(in_row * input_wd + in_col) * in_channels;
const int8_t *filter_ptr = filter_data +
out_ch_idx * in_channels * filter_ht * filter_wd +
(filter_y_idx * filter_wd + filter_x_idx) * in_channels;
int32_t in_ch_idx = 0;
for (; in_ch_idx < in_channels - 3; in_ch_idx += 4) {
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
}
for (; in_ch_idx < in_channels; in_ch_idx ++) {
conv_out += (*input_ptr++ + input_offset) * *filter_ptr++;
}
}
}
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, *out_mult++, *out_shift++);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}

57
code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c
View File

@@ -12,16 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <esp_nn_defs.h>
#include <common_functions.h>
int esp_nn_get_depthwise_conv_scratch_size_ansi(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t ch_mult,
const uint16_t filter_wd,
const uint16_t filter_ht)
int esp_nn_get_depthwise_conv_scratch_size_ansi(const data_dims_t *input_dims,
const data_dims_t *filter_dims,
const data_dims_t *output_dims,
const dw_conv_params_t *conv_params)
{
return 0;
}
@@ -31,29 +28,35 @@ void esp_nn_set_depthwise_conv_scratch_buf_ansi(const void *buf)
}
void esp_nn_depthwise_conv_s8_ansi(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
void esp_nn_depthwise_conv_s8_ansi(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
const dw_conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
const uint16_t ch_mult = conv_params->ch_mult;
int out_idx = 0;
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;

291
code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_opt.c Normal file
View File

@@ -0,0 +1,291 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <esp_nn_defs.h>
#include <common_functions.h>
int esp_nn_get_depthwise_conv_scratch_size_opt(const data_dims_t *input_dims,
const data_dims_t *filter_dims,
const data_dims_t *output_dims,
const dw_conv_params_t *conv_params)
{
return 0;
}
void esp_nn_set_depthwise_conv_scratch_buf_opt(const void *buf)
{
}
/* common channel multiplier == 1 case */
__attribute__ ((noinline))
static void esp_nn_depthwise_conv_s8_ch_mult_1(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const dw_conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
int out_idx = 0;
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;
for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
const int16_t base_x = (out_x * stride_wd) - pad_wd;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
int ch_idx = 0;
for (; ch_idx < channels - 3; ch_idx += 4) {//channel_loop
int32_t result0 = 0;
int32_t result1 = 0;
int32_t result2 = 0;
int32_t result3 = 0;
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels) + ch_idx;
int32_t input_val0 = input_data[input_index + 0] + input_offset;
int32_t input_val1 = input_data[input_index + 1] + input_offset;
int32_t input_val2 = input_data[input_index + 2] + input_offset;
int32_t input_val3 = input_data[input_index + 3] + input_offset;
int32_t filter_val0 = filter_data[filter_index + 0];
int32_t filter_val1 = filter_data[filter_index + 1];
int32_t filter_val2 = filter_data[filter_index + 2];
int32_t filter_val3 = filter_data[filter_index + 3];
result0 += input_val0 * filter_val0;
result1 += input_val1 * filter_val1;
result2 += input_val2 * filter_val2;
result3 += input_val3 * filter_val3;
}
}
if (bias) {
result0 += bias[ch_idx + 0];
result1 += bias[ch_idx + 1];
result2 += bias[ch_idx + 2];
result3 += bias[ch_idx + 3];
}
result0 = esp_nn_multiply_by_quantized_mult_fast(result0, *out_mult++, *out_shift++);
result1 = esp_nn_multiply_by_quantized_mult_fast(result1, *out_mult++, *out_shift++);
result2 = esp_nn_multiply_by_quantized_mult_fast(result2, *out_mult++, *out_shift++);
result3 = esp_nn_multiply_by_quantized_mult_fast(result3, *out_mult++, *out_shift++);
result0 += out_offset;
result1 += out_offset;
result2 += out_offset;
result3 += out_offset;
result0 = max(result0, activation_min);
result1 = max(result1, activation_min);
result2 = max(result2, activation_min);
result3 = max(result3, activation_min);
result0 = min(result0, activation_max);
result1 = min(result1, activation_max);
result2 = min(result2, activation_max);
result3 = min(result3, activation_max);
out_data[out_idx++] = result0;
out_data[out_idx++] = result1;
out_data[out_idx++] = result2;
out_data[out_idx++] = result3;
}
for (; ch_idx < channels; ch_idx++) {//channel_loop
int32_t result = 0;
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels) + ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val = filter_data[filter_index];
result += input_val * filter_val;
}
}
if (bias) {
result += bias[ch_idx];
}
result = esp_nn_multiply_by_quantized_mult_fast(result, *out_mult++, *out_shift++);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
out_data[out_idx++] = result;
}
}
}
}
void esp_nn_depthwise_conv_s8_opt(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const dw_conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t ch_mult = conv_params->ch_mult;
if (ch_mult == 1) {
esp_nn_depthwise_conv_s8_ch_mult_1(input_dims, input_data, filter_dims, filter_data,
bias, output_dims, out_data, conv_params, quant_data);
return;
}
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
int out_idx = 0;
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;
for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
const int16_t base_x = (out_x * stride_wd) - pad_wd;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
int ch_mult_idx = 0;
for (; ch_mult_idx < ch_mult - 3; ch_mult_idx += 4) {
int32_t result0 = 0;
int32_t result1 = 0;
int32_t result2 = 0;
int32_t result3 = 0;
const int out_ch_idx = ch_idx * ch_mult + ch_mult_idx;
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val0 = filter_data[filter_index + 0];
int32_t filter_val1 = filter_data[filter_index + 1];
int32_t filter_val2 = filter_data[filter_index + 2];
int32_t filter_val3 = filter_data[filter_index + 3];
result0 += input_val * filter_val0;
result1 += input_val * filter_val1;
result2 += input_val * filter_val2;
result3 += input_val * filter_val3;
}
}
if (bias) {
result0 += bias[out_ch_idx + 0];
result1 += bias[out_ch_idx + 1];
result2 += bias[out_ch_idx + 2];
result3 += bias[out_ch_idx + 3];
}
result0 = esp_nn_multiply_by_quantized_mult_fast(result0, *out_mult++, *out_shift++);
result1 = esp_nn_multiply_by_quantized_mult_fast(result1, *out_mult++, *out_shift++);
result2 = esp_nn_multiply_by_quantized_mult_fast(result2, *out_mult++, *out_shift++);
result3 = esp_nn_multiply_by_quantized_mult_fast(result3, *out_mult++, *out_shift++);
result0 += out_offset;
result1 += out_offset;
result2 += out_offset;
result3 += out_offset;
result0 = max(result0, activation_min);
result1 = max(result1, activation_min);
result2 = max(result2, activation_min);
result3 = max(result3, activation_min);
result0 = min(result0, activation_max);
result1 = min(result1, activation_max);
result2 = min(result2, activation_max);
result3 = min(result3, activation_max);
out_data[out_idx++] = result0;
out_data[out_idx++] = result1;
out_data[out_idx++] = result2;
out_data[out_idx++] = result3;
}
for (; ch_mult_idx < ch_mult; ch_mult_idx++) {
int32_t result = 0;
const int out_ch_idx = ch_idx * ch_mult + ch_mult_idx;
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val = filter_data[filter_index];
result += input_val * filter_val;
}
}
if (bias) {
result += bias[out_ch_idx];
}
result = esp_nn_multiply_by_quantized_mult_fast(result, *out_mult++, *out_shift++);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
out_data[out_idx++] = result;
}
}
}
}
}

134
code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c
View File

@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <esp_nn_defs.h>
#include <common_functions.h>
@@ -353,17 +353,59 @@ void esp_nn_depthwise_conv_s8_ch_mult1(const int8_t *input_data,
}
}
int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t ch_mult,
const uint16_t filter_wd,
const uint16_t filter_ht)
int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const data_dims_t *input_dims,
const data_dims_t *filter_dims,
const data_dims_t *output_dims,
const dw_conv_params_t *conv_params)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t channels = input_dims->channels;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t ch_mult = conv_params->ch_mult;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
int filter_size = filter_wd * filter_ht * channels * ch_mult;
int padding_used = ((filter_wd == 3) && (filter_ht == 3)) * 2;
int input_size = (input_wd + padding_used) * (input_ht + padding_used) * channels;
return 2 * (filter_size + input_size) + 16; //16 for alignment
int pad_width = 0, pad_height = 0;
if ((ch_mult == 1) && (channels % 8 == 0) && (filter_wd == 3) && (filter_ht == 3)) {
if (channels % 16 == 0) {
if (pad_wd || pad_ht) {
pad_width = pad_wd * 2;
pad_height = pad_ht * 2;
} else {
// check if we need to pad additionally
pad_width = (out_wd * stride_wd + filter_wd - 1) - input_wd;
pad_height = (out_ht * stride_ht + filter_ht - 1) - input_ht;
// printf("in(%d %d %d), out(%d %d), filter (%d %d) stride (%d %d), pad (%d %d)",
// input_wd, input_ht, channels, out_wd, out_ht, filter_wd, filter_ht,
// stride_wd, stride_ht, pad_wd, pad_ht);
}
if (pad_width || pad_height) {
int input_size = (input_wd + pad_width) * (input_ht + pad_height) * channels;
// printf("ask1 %d\n", filter_size + input_size + 16);
return filter_size + input_size + 16; // 16 for alignment
} else {
// printf("ask2 %d\n", filter_size + 16);
return filter_size + 16; // 16 for alignment
}
} else {
int input_size = input_wd * input_ht * channels;
// printf("ask3 %d\n", 2 * (filter_size + input_size) + 16);
return 2 * (filter_size + input_size) + 16; // 16 for alignment
}
} else if (ch_mult % 4 == 0) {
int input_size = input_wd * input_ht * channels;
// printf("ask4 %d\n", 2 * (filter_size + input_size) + 16);
return 2 * (filter_size + input_size) + 16; // 16 for alignment
}
return 32; // just few bytes
}
void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(void *buf)
@@ -376,29 +418,38 @@ void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(void *buf)
* Assumption 2: Pointers are valid
* Assumption 3: dialation width = 1
*/
void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
void esp_nn_depthwise_conv_s8_esp32s3(const data_dims_t *input_dims,
const int8_t *input_data,
const data_dims_t *filter_dims,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
const data_dims_t *output_dims,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
const dw_conv_params_t *conv_params,
const quant_data_t *quant_data)
{
const uint16_t input_wd = input_dims->width;
const uint16_t input_ht = input_dims->height;
const uint16_t channels = input_dims->channels;
const int32_t input_offset = conv_params->in_offset;
const int32_t out_offset = conv_params->out_offset;
const uint16_t pad_wd = conv_params->padding.width;
const uint16_t pad_ht = conv_params->padding.height;
const uint16_t stride_wd = conv_params->stride.width;
const uint16_t stride_ht = conv_params->stride.height;
const uint16_t filter_wd = filter_dims->width;
const uint16_t filter_ht = filter_dims->height;
const uint16_t out_wd = output_dims->width;
const uint16_t out_ht = output_dims->height;
const int32_t *out_shift = quant_data->shift;
const int32_t *out_mult = quant_data->mult;
const int32_t activation_min = conv_params->activation.min;
const int32_t activation_max = conv_params->activation.max;
const uint16_t ch_mult = conv_params->ch_mult;
int filter_size = filter_wd * filter_ht * channels * ch_mult;
int align_len = 16 - (filter_size & 15);
int input_size = input_wd * input_ht * channels;
@@ -423,18 +474,27 @@ void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data,
stride_wd, stride_ht, filter_aligned, bias,
out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else if ((pad_wd == 0) && (pad_ht == 0) &&
// because this does not handle padding offset cases yet, run just for stride (1, 1).
// end padding of input with `-input_offset` should solve this
(stride_wd == 1) && (stride_ht == 1)) {
} else if ((channels % 16 == 0) && (pad_wd == 0) && (pad_ht == 0)) {
/* process in 8 bits */
int8_t *filter_aligned = (int8_t *) scratch_buffer;
int8_t *input_padded = (int8_t *) scratch_buffer + filter_size + align_len;
// check if we need to pad additionally
int pad_right = (out_wd * stride_wd + filter_wd - 1) - input_wd;
int pad_bottom = (out_ht * stride_ht + filter_ht - 1) - input_ht;
if (pad_right || pad_bottom) { // pad right and bottom
esp_nn_aligned_s8_pad_end_with_value(input_data, input_padded, input_wd, input_ht,
channels, -input_offset, pad_right, pad_bottom);
} else {
input_padded = (int8_t *) input_data;
}
memcpy(filter_aligned, filter_data, filter_size);
esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_data, input_wd, input_ht, channels, input_offset,
stride_wd, stride_ht, filter_aligned,
bias, out_data, out_wd, out_ht, out_offset, out_shift,
esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_padded, input_wd + pad_right,
input_ht + pad_bottom, channels, input_offset,
stride_wd, stride_ht, filter_aligned, bias,
out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else { /* (channels % 8) == 0 && pad_wd == 1 && pad_ht == 1 */
} else { /* (channels % 8) == 0 */
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3(input_data16, input_wd, input_ht, channels,