Mirror/AI-on-the-edge-device
1
0
Fork 0
mirror of https://github.com/jomjol/AI-on-the-edge-device.git synced 2025-12-09 04:56:53 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rolling 20220526

Browse Source
This commit is contained in:
jomjol
2022-05-26 20:31:26 +02:00
parent cce812ff11
commit 00028010ee
203 changed files with 12003 additions and 1226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
code/components/esp-nn/src/activation_functions/esp_nn_relu_ansi.c Normal file
View File

@@ -0,0 +1,30 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdlib.h>
#include <common_functions.h>
void esp_nn_relu6_s8_ansi(int8_t *data, uint16_t size)
{
int32_t i;
for (i = 0; i < size; i++) {
int32_t ip = data[i];
ip = max(ip, 0);
data[i] = min(ip, 6);
}
}

97
code/components/esp-nn/src/basic_math/esp_nn_add_ansi.c Normal file
View File

@@ -0,0 +1,97 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
void esp_nn_add_elementwise_u8_ansi(const uint8_t *input1_data,
const uint8_t *input2_data,
const int32_t input1_offset,
const int32_t input2_offset,
const int32_t input1_mult,
const int32_t input2_mult,
const int32_t input1_shift,
const int32_t input2_shift,
const int32_t left_shift,
uint8_t *output,
const int32_t out_offset,
const int32_t out_mult,
const int32_t out_shift,
const int32_t activation_min,
const int32_t activation_max,
const int32_t size)
{
for (int i = 0; i < size; i++) {
int32_t tmp1 = input1_data[i] + input1_offset;
int32_t tmp2 = input2_data[i] + input2_offset;
tmp1 <<= left_shift;
tmp2 <<= left_shift;
tmp1 = esp_nn_sat_round_doubling_high_mul(tmp1, input1_mult);
tmp2 = esp_nn_sat_round_doubling_high_mul(tmp2, input2_mult);
tmp1 = esp_nn_div_by_power_of_two(tmp1, -input1_shift);
tmp2 = esp_nn_div_by_power_of_two(tmp2, -input2_shift);
int32_t out = tmp1 + tmp2;
out = esp_nn_sat_round_doubling_high_mul(out, out_mult);
out = esp_nn_div_by_power_of_two(out, -out_shift);
out = out + out_offset;
out = max(activation_min, min(out, activation_max));
output[i] = (uint8_t) out;
}
}
void esp_nn_add_elementwise_s8_ansi(const int8_t *input1_data,
const int8_t *input2_data,
const int32_t input1_offset,
const int32_t input2_offset,
const int32_t input1_mult,
const int32_t input2_mult,
const int32_t input1_shift,
const int32_t input2_shift,
const int32_t left_shift,
int8_t *output,
const int32_t out_offset,
const int32_t out_mult,
const int32_t out_shift,
const int32_t activation_min,
const int32_t activation_max,
const int32_t size)
{
for (int i = 0; i < size; i++) {
int32_t tmp1 = input1_data[i] + input1_offset;
int32_t tmp2 = input2_data[i] + input2_offset;
tmp1 <<= left_shift;
tmp2 <<= left_shift;
tmp1 = esp_nn_sat_round_doubling_high_mul(tmp1, input1_mult);
tmp2 = esp_nn_sat_round_doubling_high_mul(tmp2, input2_mult);
tmp1 = esp_nn_div_by_power_of_two(tmp1, -input1_shift);
tmp2 = esp_nn_div_by_power_of_two(tmp2, -input2_shift);
int32_t out = tmp1 + tmp2;
out = esp_nn_sat_round_doubling_high_mul(out, out_mult);
out = esp_nn_div_by_power_of_two(out, -out_shift);
out = out + out_offset;
out = max(activation_min, min(out, activation_max));
output[i] = (int8_t) out;
}
}

42
code/components/esp-nn/src/basic_math/esp_nn_mul_ansi.c Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
void esp_nn_mul_elementwise_s8_ansi(const int8_t *input1_data,
const int8_t *input2_data,
const int32_t input1_offset,
const int32_t input2_offset,
int8_t *output,
const int32_t out_offset,
const int32_t out_mult,
const int32_t out_shift,
const int32_t activation_min,
const int32_t activation_max,
const int32_t size)
{
for (int i = 0; i < size; i++) {
int32_t tmp1 = input1_data[i] + input1_offset;
int32_t tmp2 = input2_data[i] + input2_offset;
int32_t out = tmp1 * tmp2;
out = esp_nn_multiply_by_quantized_mult(out, out_mult, out_shift);
out = out + out_offset;
out = max(activation_min, min(out, activation_max));
output[i] = (int8_t) out;
}
}

218
code/components/esp-nn/src/common/common_functions.h Normal file
View File

@@ -0,0 +1,218 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
/**
* c99 standard still doesn't strictly inline functions
* We need to use attribute as well to do this.
*/
#define __NN_FORCE_INLINE__ __attribute((always_inline)) static inline
/* min/max macros */
#ifndef max
#define max(a, b) ({ \
__typeof__ (a) _a = (a); \
__typeof__ (b) _b = (b); \
_a > _b ? _a : _b; \
})
#define min(a, b) ({ \
__typeof__ (a) _a = (a); \
__typeof__ (b) _b = (b); \
_a < _b ? _a : _b; \
})
#endif
__NN_FORCE_INLINE__ int32_t esp_nn_clz32(uint32_t in)
{
__asm__ volatile("nsau %0, %0" : "+r" (in));
return in;
}
__NN_FORCE_INLINE__ int32_t esp_nn_pick_sat_high32_of64(int64_t val64)
{
int32_t sign = (int32_t) (val64 >> 63);
int32_t to_add = sign & ((1ul << 31) - 1);
return (int32_t) ((int64_t) (val64 + to_add) >> 31);
}
/**
* Signed saturate a 32 bit value to 8 bits keeping output in 32 bit variable.
*/
__NN_FORCE_INLINE__ int32_t esp_nn_saturate8(int32_t in)
{
__asm__ volatile("clamps %0, %0, 7" : "+a"(in));
return in;
}
__NN_FORCE_INLINE__ int32_t esp_nn_sat_round_doubling_high_mul(int32_t in0, int32_t in1)
{
int32_t result;
int64_t in0_64 = (int64_t) in0;
bool overflow = (in0 == in1) && (in0 == (int32_t) INT32_MIN);
/* Nudge value */
int64_t nudge_val = 1 << 30;
if ((in0 < 0) ^ (in1 < 0)) {
nudge_val = 1 - nudge_val;
}
/* Multiply and add nudge */
int64_t mult = in0_64 * in1 + nudge_val;
/* Round and pickup 32 bits */
result = esp_nn_pick_sat_high32_of64(mult);
return overflow ? INT32_MAX : result;
}
/**
* fast version
* this will fail for values closer to INT32_MAX and INT32_MIN by `1 << (exponent - 1)`.
* We can afford to do this because we are at the very last stage of filter.
* Also it is pretty rare condition as our output is going to be 8 bit.
*/
__NN_FORCE_INLINE__ int32_t esp_nn_div_by_power_of_two_fast(int32_t val, int32_t exponent)
{
int32_t to_add = (1 << (exponent - 1)) - (val < 0);
return (int32_t) ((val + to_add) >> exponent);
}
__NN_FORCE_INLINE__ int32_t esp_nn_div_by_power_of_two(int32_t val, int32_t exponent)
{
int32_t result;
const int32_t mask = (1 << exponent) - 1;
const int32_t remainder = val & mask;
result = val >> exponent;
int32_t threshold = (mask >> 1) + (result < 0);
if (remainder > threshold) {
result += 1;
}
return result;
}
__NN_FORCE_INLINE__ int32_t esp_nn_multiply_by_quantized_mult(int32_t x, int32_t mult, int32_t shift)
{
int32_t left_shift = shift > 0 ? shift : 0;
int32_t right_shift = shift > 0 ? 0 : -shift;
int32_t result = esp_nn_sat_round_doubling_high_mul(x * (1 << left_shift), mult);
return esp_nn_div_by_power_of_two(result, right_shift);
}
__NN_FORCE_INLINE__ int32_t esp_nn_multiply_by_quantized_mult_fast(int32_t x, int32_t mult, int32_t shift)
{
int32_t left_shift = max(shift, 0);
int32_t right_shift = left_shift - shift;
int64_t nudge_val = 1 << 30;
int64_t in0_64 = (int64_t) (x << left_shift);
/* Multiply and add nudge */
int64_t mult_64 = in0_64 * mult + nudge_val;
int32_t result = (int32_t) (mult_64 >> 31);
if (right_shift) {
result = esp_nn_div_by_power_of_two_fast(result, right_shift);
}
return result;
}
static void esp_nn_aligned_s8_pad_with_value(const int8_t *src, int8_t *dst,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t pad_val,
const uint16_t pad_wd,
const uint16_t pad_ht)
{
/* memset with pad_val */
memset(dst, pad_val, ((input_wd + 2 * pad_wd) * (input_ht + 2 * pad_ht)) * channels * 2);
dst += (pad_wd + input_wd + pad_wd) * channels;
for (int i = 0; i < input_ht; i++) {
dst += pad_wd * channels;
for (int j = 0; j < input_wd * channels; j++) {
*dst++ = *src++;
}
dst += pad_wd * channels;
}
}
#if 0
static void esp_nn_aligned_s8_pad_end_with_value(const int8_t *src, int8_t *dst,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t pad_val,
const uint16_t pad_wd,
const uint16_t pad_ht)
{
for (int i = 0; i < input_ht; i++) {
for (int j = 0; j < input_wd * channels; j++) {
*dst++ = *src++;
}
memset(dst, pad_val, pad_wd * channels);
dst += pad_wd * channels;
}
/* pad end `pad_ht` lines at end */
memset(dst, pad_val, (input_wd + pad_wd) * pad_ht * channels);
}
#endif
/**
* @brief convert 8 bit input data to 16 bit
*
* @param src int8_t source data
* @param dst int16_t dst data
* @param size length of data
* @param offset offset to be added to src data. Range: [-128, 127]
*/
__NN_FORCE_INLINE__ void esp_nn_s8_to_s16_with_offset(const int8_t *src, int16_t *dst,
const int size, const int32_t offset)
{
int i = 0;
for (; i < size; i += 2) {
dst[i + 0] = src[i + 0] + offset;
dst[i + 1] = src[i + 1] + offset;
}
if(i < size) {
dst[i] = src[i] + offset;
}
}
/**
* @brief convert 8 bit input data to 16 bit
*
* @param src int8_t source data
* @param dst int16_t dst data
* @param size length of data
*/
__NN_FORCE_INLINE__ void esp_nn_s8_to_s16(const int8_t *src, int16_t *dst, const int size)
{
int i = 0;
for (; i < size; i += 2) {
dst[i + 0] = src[i + 0];
dst[i + 1] = src[i + 1];
}
if(i < size) {
dst[i] = src[i];
}
}

175
code/components/esp-nn/src/convolution/esp_nn_conv_ansi.c Normal file
View File

@@ -0,0 +1,175 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
int esp_nn_get_conv_scratch_size_ansi(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_ch,
const uint16_t out_ch,
const uint16_t filter_wd,
const uint16_t filter_ht)
{
return 0;
}
void esp_nn_set_conv_scratch_buf_ansi(const void *buf)
{
}
/**
* Assumption 1: i/p channels == o/p channels
* Assumption 2: Pointers are valid
* Assumption 3: dialation width = 1
*/
void esp_nn_conv_u8_ansi(const uint8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t filter_offset,
const int32_t *bias,
uint8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t out_shift,
const int32_t out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;
for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
const int16_t base_x = (out_x * stride_wd) - pad_wd;
for (int out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {//channel_loop
int32_t result = 0;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
for (int in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
int32_t input_index = (idx_y * input_wd + idx_x) * in_channels + in_ch_idx;
int32_t filter_index = ((out_ch_idx * filter_ht + filter_y_idx)
* filter_wd + filter_x_idx) * in_channels
+ in_ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val = filter_data[filter_index] + filter_offset;
result += input_val * filter_val;
}
}
}
if (bias) {
result += bias[out_ch_idx];
}
result = esp_nn_multiply_by_quantized_mult(result, out_mult, out_shift);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
int out_index = (out_y * out_wd + out_x) * out_channels + out_ch_idx;
out_data[out_index] = (uint8_t) result;
}
}
}
}
/**
* Assumption 1: i/p channels == o/p channels
* Assumption 2: Pointers are valid
* Assumption 3: dialation width = 1
*/
void esp_nn_conv_s8_ansi(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
const int32_t base_y = stride_ht * out_y - pad_ht;
const int32_t base_x = stride_wd * out_x - pad_wd;
const int32_t filter_y_start = max(0, -base_y);
const int32_t filter_x_start = max(0, -base_x);
const int32_t filter_y_end = min(filter_ht, input_ht - base_y);
const int32_t filter_x_end = min(filter_wd, input_wd - base_x);
for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t in_row = base_y + filter_y_idx;
const int32_t in_col = base_x + filter_x_idx;
int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd +
(filter_y_idx * filter_wd + filter_x_idx) * in_channels;
for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
conv_out +=
(input_data[input_base_offset + in_ch_idx] + input_offset) *
filter_data[filter_base_offset + in_ch_idx];
}
}
}
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}

436
code/components/esp-nn/src/convolution/esp_nn_conv_esp32s3.c Normal file
View File

@@ -0,0 +1,436 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <common_functions.h>
static int16_t *scratch_buffer = NULL;
extern void esp_nn_conv_s16_mult8_1x1_esp32s3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const int16_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max,
void *buffer /* scratch buffer */);
extern void esp_nn_conv_s16_mult4_1x1_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int16_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max,
void *buffer /* scratch buffer */);
extern void esp_nn_conv_s16_mult8_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int16_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_aligned_s8_to_s16_with_offset_esp32s3(const int8_t *src, int16_t *dst,
const int size, const int32_t offset);
extern void esp_nn_s8_to_s16_esp32s3(const int8_t *src, int16_t *dst, const int size);
static void esp_nn_conv_s8_unrolled(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
const int32_t base_y = stride_ht * out_y - pad_ht;
const int32_t base_x = stride_wd * out_x - pad_wd;
const int32_t filter_y_start = max(0, -base_y);
const int32_t filter_x_start = max(0, -base_x);
const int32_t filter_y_end = min(filter_ht, input_ht - base_y);
const int32_t filter_x_end = min(filter_wd, input_wd - base_x);
for (filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
for (filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t in_row = base_y + filter_y_idx;
const int32_t in_col = base_x + filter_x_idx;
int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd +
(filter_y_idx * filter_wd + filter_x_idx) * in_channels;
for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
conv_out +=
(input_data[input_base_offset + in_ch_idx] + input_offset) *
filter_data[filter_base_offset + in_ch_idx];
}
}
}
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}
static void esp_nn_conv_s8_pad_valid(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
const int32_t base_y = stride_ht * out_y;
const int32_t base_x = stride_wd * out_x;
for (filter_y_idx = 0; filter_y_idx < filter_ht; filter_y_idx++) {
for (filter_x_idx = 0; filter_x_idx < filter_wd; filter_x_idx++) {
const int32_t in_row = base_y + filter_y_idx;
const int32_t in_col = base_x + filter_x_idx;
int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
int32_t filter_base_offset = out_ch_idx * in_channels * filter_ht * filter_wd +
(filter_y_idx * filter_wd + filter_x_idx) * in_channels;
const int8_t *input_data_ptr = input_data + input_base_offset;
const int8_t *filter_data_ptr = filter_data + filter_base_offset;
for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
conv_out += (*input_data_ptr++ + input_offset) * *filter_data_ptr++;
}
}
}
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}
static void esp_nn_conv_s8_pad_valid_3x3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_channels,
const int32_t input_offset,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int32_t out_ch_idx, out_y, out_x, in_ch_idx, filter_y_idx, filter_x_idx;
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
const int32_t base_y = stride_ht * out_y;
const int32_t base_x = stride_wd * out_x;
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
for (filter_y_idx = 0; filter_y_idx < 3; filter_y_idx++) {
for (filter_x_idx = 0; filter_x_idx < 3; filter_x_idx++) {
const int32_t in_row = base_y + filter_y_idx;
const int32_t in_col = base_x + filter_x_idx;
int32_t input_base_offset = (in_row * input_wd + in_col) * in_channels;
int32_t filter_base_offset = out_ch_idx * in_channels * 3 * 3 +
(filter_y_idx * 3 + filter_x_idx) * in_channels;
const int8_t *input_data_ptr = input_data + input_base_offset;
const int8_t *filter_data_ptr = filter_data + filter_base_offset;
for (in_ch_idx = 0; in_ch_idx < in_channels; in_ch_idx++) {
conv_out += (*input_data_ptr++ + input_offset) * *filter_data_ptr++;
}
}
}
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}
static void esp_nn_conv_s8_pad_valid_ch3_3x3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const int32_t input_offset,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int32_t out_ch_idx, out_y, out_x, filter_y_idx;
/* use scratch_buffer to pre-compute offset factor */
int16_t *filter_sum = (int16_t *) scratch_buffer;
const int8_t *filter_ptr = filter_data;
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int16_t sum_val = 0;
for (int i = 0; i < 9; i++) {
sum_val += *filter_ptr++;
sum_val += *filter_ptr++;
sum_val += *filter_ptr++;
}
*filter_sum++ = sum_val;
}
for (out_y = 0; out_y < out_ht; out_y++) {
for (out_x = 0; out_x < out_wd; out_x++) {
const int8_t *filter_data_ptr = filter_data;
const int32_t base_y = stride_ht * out_y;
const int32_t base_x = stride_wd * out_x;
const int8_t *input_base_ptr = input_data + (base_y * input_wd + base_x) * 3;
int16_t *filter_sum = (int16_t *) scratch_buffer;
for (out_ch_idx = 0; out_ch_idx < out_channels; out_ch_idx++) {
int32_t conv_out = 0;
for (filter_y_idx = 0; filter_y_idx < 3; filter_y_idx++) {
const int8_t *input_data_ptr = input_base_ptr + (filter_y_idx * input_wd) * 3;
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
conv_out += (*input_data_ptr++) * (*filter_data_ptr++);
}
conv_out += *filter_sum++ * input_offset;
if (bias) {
conv_out += bias[out_ch_idx];
}
conv_out = esp_nn_multiply_by_quantized_mult_fast(conv_out, out_mult[out_ch_idx], out_shift[out_ch_idx]);
conv_out += out_offset;
conv_out = max(conv_out, activation_min);
conv_out = min(conv_out, activation_max);
*out_data++ = (int8_t) conv_out;
}
}
}
}
int esp_nn_get_conv_scratch_size_esp32s3(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t in_ch,
const uint16_t out_ch,
const uint16_t filter_wd,
const uint16_t filter_ht)
{
int filter_size = filter_wd * filter_ht * in_ch * out_ch;
int input_size = input_wd * input_ht * in_ch;
int transpose_buf_size = 8 * in_ch; /* to store intermediate data */
int align_buf_size = 32; /* extra buffer for alignment */
return 2 * (filter_size + input_size + transpose_buf_size) + align_buf_size;
}
void esp_nn_set_conv_scratch_buf_esp32s3(void *buf)
{
scratch_buffer = (int16_t *) buf;
}
void esp_nn_conv_s8_esp32s3(const int8_t *input,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int filter_size = filter_wd * filter_ht * channels * out_channels;
int input_size = input_wd * input_ht * channels;
int align_len = 16 - (filter_size & 15);
int16_t *filter_data16 = scratch_buffer;
int16_t *input_data16 = scratch_buffer + filter_size + align_len;
if (scratch_buffer == NULL) {
printf("esp_nn_conv error! scratch_buffer not set!\n");
return;
}
if (channels % 8 == 0 && filter_wd == 1 && filter_ht == 1 &&
pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
int scratch_offset = (int) (filter_data16 + filter_size);
void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15));
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_conv_s16_mult8_1x1_esp32s3(
input, input_wd, input_ht, channels, input_offset, filter_data16,
bias, out_data, out_wd, out_ht, out_channels, out_offset,
out_shift, out_mult, activation_min, activation_max, scratch_buf);
} else if (channels % 4 == 0 && filter_wd == 1 && filter_ht == 1 &&
(input_wd * input_ht) % 16 == 0 && /* TODO: remove this check */
pad_wd == 0 && pad_ht == 0 && stride_wd == 1 && stride_ht == 1) {
int scratch_offset = (int) (input_data16 + input_size);
void *scratch_buf = (void *) (scratch_offset + 16 - (scratch_offset & 15));
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input, input_data16, input_size, input_offset);
esp_nn_conv_s16_mult4_1x1_esp32s3(
input_data16, input_wd, input_ht, channels, filter_data16,
bias, out_data, out_wd, out_ht, out_channels, out_offset,
out_shift, out_mult, activation_min, activation_max, scratch_buf);
} else if (channels % 8 == 0) {
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input, input_data16, input_size, input_offset);
esp_nn_conv_s16_mult8_esp32s3(
input_data16, input_wd, input_ht, channels, pad_wd, pad_ht,
stride_wd, stride_ht, filter_data16, filter_wd, filter_ht, bias,
out_data, out_wd, out_ht, out_channels, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else if (pad_wd == 0 && pad_ht == 0) {
if (filter_wd == 3 && filter_ht == 3 && channels == 3) {
esp_nn_conv_s8_pad_valid_ch3_3x3(input, input_wd, input_ht, input_offset,
stride_wd, stride_ht, filter_data, bias,
out_data, out_wd, out_ht, out_channels, out_offset,
out_shift, out_mult, activation_min, activation_max);
} else {
esp_nn_conv_s8_pad_valid(input, input_wd, input_ht, channels, input_offset,
stride_wd, stride_ht, filter_data, filter_wd, filter_ht, bias,
out_data, out_wd, out_ht, out_channels, out_offset, out_shift,
out_mult, activation_min, activation_max);
}
} else {
/* Basic unrolled version */
esp_nn_conv_s8_unrolled(input, input_wd, input_ht, channels, input_offset,
pad_wd, pad_ht, stride_wd, stride_ht,
filter_data, filter_wd, filter_ht, bias,
out_data, out_wd, out_ht, out_channels, out_offset, out_shift,
out_mult, activation_min, activation_max);
}
}

97
code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_ansi.c Normal file
View File

@@ -0,0 +1,97 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
int esp_nn_get_depthwise_conv_scratch_size_ansi(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t ch_mult,
const uint16_t filter_wd,
const uint16_t filter_ht)
{
return 0;
}
void esp_nn_set_depthwise_conv_scratch_buf_ansi(const void *buf)
{
}
void esp_nn_depthwise_conv_s8_ansi(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int out_idx = 0;
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;
for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
const int16_t base_x = (out_x * stride_wd) - pad_wd;
for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
for (int ch_mult_idx = 0; ch_mult_idx < ch_mult; ch_mult_idx++) {
int32_t result = 0;
const int out_ch_idx = ch_mult_idx + ch_idx * ch_mult;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val = filter_data[filter_index];
result += input_val * filter_val;
}
}
if (bias) {
result += bias[out_ch_idx];
}
result = esp_nn_multiply_by_quantized_mult(result, out_mult[out_ch_idx], out_shift[out_ch_idx]);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
out_data[out_idx++] = result;
}
}
}
}
}

483
code/components/esp-nn/src/convolution/esp_nn_depthwise_conv_s8_esp32s3.c Normal file
View File

@@ -0,0 +1,483 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <stdio.h>
#include <common_functions.h>
static int16_t *scratch_buffer = NULL;
extern void esp_nn_depthwise_conv_s16_mult8_3x3_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
const int16_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_depthwise_conv_s16_mult1_3x3_no_pad_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int16_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_depthwise_conv_s16_mult8_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
const int16_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_depthwise_conv_s16_mult4_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
const int16_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int16_t *filter_data,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_depthwise_conv_s16_mult1_esp32s3(const int16_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int16_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max);
extern void esp_nn_s8_to_s16_esp32s3(const int8_t *src, int16_t *dst, const int size);
extern void esp_nn_aligned_s8_to_s16_with_offset_esp32s3(const int8_t *src, int16_t *dst,
const int size, const int32_t offset);
static void esp_nn_depthwise_conv_s8_unrolled(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int out_idx = 0;
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;
for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
const int16_t base_x = (out_x * stride_wd) - pad_wd;
for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
int ch_mult_idx = 0;
for (; ch_mult_idx < ch_mult - 3; ch_mult_idx += 4) {
int32_t result0 = 0, result1 = 0, result2 = 0, result3 = 0;
const int out_ch_idx = ch_mult_idx + ch_idx * ch_mult;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val0 = filter_data[filter_index + 0];
int32_t filter_val1 = filter_data[filter_index + 1];
int32_t filter_val2 = filter_data[filter_index + 2];
int32_t filter_val3 = filter_data[filter_index + 3];
result0 += input_val * filter_val0;
result1 += input_val * filter_val1;
result2 += input_val * filter_val2;
result3 += input_val * filter_val3;
}
}
if (bias) {
result0 += bias[out_ch_idx + 0];
result1 += bias[out_ch_idx + 1];
result2 += bias[out_ch_idx + 2];
result3 += bias[out_ch_idx + 3];
}
result0 = esp_nn_multiply_by_quantized_mult(result0,
out_mult[out_ch_idx + 0], out_shift[out_ch_idx + 0]);
result1 = esp_nn_multiply_by_quantized_mult(result1,
out_mult[out_ch_idx + 1], out_shift[out_ch_idx + 1]);
result2 = esp_nn_multiply_by_quantized_mult(result2,
out_mult[out_ch_idx + 2], out_shift[out_ch_idx + 2]);
result3 = esp_nn_multiply_by_quantized_mult(result3,
out_mult[out_ch_idx + 3], out_shift[out_ch_idx + 3]);
result0 += out_offset;
result1 += out_offset;
result2 += out_offset;
result3 += out_offset;
result0 = max(result0, activation_min);
result1 = max(result1, activation_min);
result2 = max(result2, activation_min);
result3 = max(result3, activation_min);
result0 = min(result0, activation_max);
result1 = min(result1, activation_max);
result2 = min(result2, activation_max);
result3 = min(result3, activation_max);
out_data[out_idx++] = result0;
out_data[out_idx++] = result1;
out_data[out_idx++] = result2;
out_data[out_idx++] = result3;
}
/* left-over */
for (; ch_mult_idx < ch_mult; ch_mult_idx++) {
int32_t result = 0;
const int out_ch_idx = ch_mult_idx + ch_idx * ch_mult;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * (channels * ch_mult) + out_ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val = filter_data[filter_index];
result += input_val * filter_val;
}
}
if (bias) {
result += bias[out_ch_idx];
}
result = esp_nn_multiply_by_quantized_mult(result, out_mult[out_ch_idx], out_shift[out_ch_idx]);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
out_data[out_idx++] = result;
}
}
}
}
}
void esp_nn_depthwise_conv_s8_ch_mult1(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int out_idx = 0;
for (int out_y = 0; out_y < out_ht; out_y++) { //height loop
const int16_t base_y = (out_y * stride_ht) - pad_ht;
for (int out_x = 0; out_x < out_wd; out_x++) { //width_loop
const int16_t base_x = (out_x * stride_wd) - pad_wd;
for (int ch_idx = 0; ch_idx < channels; ch_idx++) {//channel_loop
int32_t result = 0;
/* Select filter so as the point doesn't lie outside block */
int filter_y_start = max(0, -base_y);
int filter_x_start = max(0, -base_x);
int filter_y_end = min(filter_ht, input_ht - base_y);
int filter_x_end = min(filter_wd, input_wd - base_x);
for (int filter_y_idx = filter_y_start; filter_y_idx < filter_y_end; filter_y_idx++) {
const int32_t idx_y = base_y + filter_y_idx;
for (int filter_x_idx = filter_x_start; filter_x_idx < filter_x_end; filter_x_idx++) {
const int32_t idx_x = base_x + filter_x_idx;
int32_t input_index = (idx_y * input_wd + idx_x) * channels + ch_idx;
int32_t filter_index = (filter_y_idx * filter_wd + filter_x_idx) * channels + ch_idx;
int32_t input_val = input_data[input_index] + input_offset;
int32_t filter_val = filter_data[filter_index];
result += input_val * filter_val;
}
}
if (bias) {
result += bias[ch_idx];
}
result = esp_nn_multiply_by_quantized_mult(result, out_mult[ch_idx], out_shift[ch_idx]);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
out_data[out_idx++] = result;
}
}
}
}
int esp_nn_get_depthwise_conv_scratch_size_esp32s3(const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const uint16_t ch_mult,
const uint16_t filter_wd,
const uint16_t filter_ht)
{
int filter_size = filter_wd * filter_ht * channels * ch_mult;
int padding_used = ((filter_wd == 3) && (filter_ht == 3)) * 2;
int input_size = (input_wd + padding_used) * (input_ht + padding_used) * channels;
return 2 * (filter_size + input_size) + 16; //16 for alignment
}
void esp_nn_set_depthwise_conv_scratch_buf_esp32s3(void *buf)
{
scratch_buffer = (int16_t *) buf;
}
/**
* Assumption 1: i/p channels == o/p channels
* Assumption 2: Pointers are valid
* Assumption 3: dialation width = 1
*/
void esp_nn_depthwise_conv_s8_esp32s3(const int8_t *input_data,
const uint16_t input_wd,
const uint16_t input_ht,
const uint16_t channels,
const int32_t input_offset,
const uint16_t pad_wd,
const uint16_t pad_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t ch_mult,
const int8_t *filter_data,
const uint16_t filter_wd,
const uint16_t filter_ht,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_wd,
const uint16_t out_ht,
const int32_t out_offset,
const int32_t *out_shift,
const int32_t *out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
int filter_size = filter_wd * filter_ht * channels * ch_mult;
int align_len = 16 - (filter_size & 15);
int input_size = input_wd * input_ht * channels;
int16_t *filter_data16 = scratch_buffer;
int16_t *input_data16 = scratch_buffer + filter_size + align_len;
if (scratch_buffer == NULL) {
printf("esp_nn_depthwise_conv error! scratch_buffer not set!\n");
return;
}
if ((ch_mult == 1) && (channels % 8 == 0)) {
if ((filter_wd == 3) && (filter_ht == 3)) {
if ((channels % 16 == 0) && (pad_wd == 1) && (pad_ht == 1)) {
/* process in 8 bits */
int8_t *filter_aligned = (int8_t *) scratch_buffer;
int8_t *input_padded = (int8_t *) scratch_buffer + filter_size + align_len;
memcpy(filter_aligned, filter_data, filter_size);
esp_nn_aligned_s8_pad_with_value(input_data, input_padded, input_wd, input_ht, channels,
-input_offset, pad_wd, pad_ht);
esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_padded, input_wd + 2 * pad_wd,
input_ht + 2 * pad_ht, channels, input_offset,
stride_wd, stride_ht, filter_aligned, bias,
out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else if ((pad_wd == 0) && (pad_ht == 0) &&
// because this does not handle padding offset cases yet, run just for stride (1, 1).
// end padding of input with `-input_offset` should solve this
(stride_wd == 1) && (stride_ht == 1)) {
/* process in 8 bits */
int8_t *filter_aligned = (int8_t *) scratch_buffer;
memcpy(filter_aligned, filter_data, filter_size);
esp_nn_depthwise_conv_s8_mult1_3x3_padded_esp32s3(input_data, input_wd, input_ht, channels, input_offset,
stride_wd, stride_ht, filter_aligned,
bias, out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else { /* (channels % 8) == 0 && pad_wd == 1 && pad_ht == 1 */
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
esp_nn_depthwise_conv_s16_mult1_3x3_esp32s3(input_data16, input_wd, input_ht, channels,
pad_wd, pad_ht, stride_wd, stride_ht, filter_data16,
bias, out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
}
} else { // all other ch_mult == 1, `channels % 8 == 0`
esp_nn_depthwise_conv_s8_ch_mult1(input_data, input_wd, input_ht, channels, input_offset,
pad_wd, pad_ht, stride_wd, stride_ht,
filter_data, filter_wd, filter_ht,
bias, out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
}
} else if (ch_mult % 8 == 0) {
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
if (filter_wd == 3 && filter_ht == 3) {
esp_nn_depthwise_conv_s16_mult8_3x3_esp32s3(input_data16, input_wd, input_ht, channels,
pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
filter_data16, bias,
out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else {
esp_nn_depthwise_conv_s16_mult8_esp32s3(input_data16, input_wd, input_ht, channels,
pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
filter_data16, filter_wd, filter_ht, bias,
out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
}
} else if (ch_mult % 4 == 0) {
esp_nn_s8_to_s16_esp32s3(filter_data, filter_data16, filter_size);
esp_nn_aligned_s8_to_s16_with_offset_esp32s3(input_data, input_data16, input_size, input_offset);
esp_nn_depthwise_conv_s16_mult4_esp32s3(input_data16, input_wd, input_ht, channels,
pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
filter_data16, filter_wd, filter_ht, bias,
out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
} else {
esp_nn_depthwise_conv_s8_unrolled(input_data, input_wd, input_ht, channels, input_offset,
pad_wd, pad_ht, stride_wd, stride_ht, ch_mult,
filter_data, filter_wd, filter_ht,
bias, out_data, out_wd, out_ht, out_offset, out_shift,
out_mult, activation_min, activation_max);
}
}

50
code/components/esp-nn/src/fully_connected/esp_nn_fully_connected_ansi.c Normal file
View File

@@ -0,0 +1,50 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
void esp_nn_fully_connected_s8_ansi(const int8_t *input_data,
const int32_t input_offset,
const uint16_t row_len,
const int8_t *filter_data,
const int32_t filter_offset,
const int32_t *bias,
int8_t *out_data,
const uint16_t out_channels,
const int32_t out_offset,
const int32_t out_shift,
const int32_t out_mult,
const int32_t activation_min,
const int32_t activation_max)
{
for (int32_t out_c = 0; out_c < out_channels; ++out_c) {
int32_t result = 0;
for (int32_t data_idx = 0; data_idx < row_len; data_idx++) {
int32_t filter_index = row_len * out_c + data_idx;
int32_t input_val = input_data[data_idx];
int32_t filter_val = filter_data[filter_index];
result += (filter_val + filter_offset) * (input_val + input_offset);
}
if (bias) {
result += bias[out_c];
}
result = esp_nn_multiply_by_quantized_mult(result, out_mult, out_shift);
result += out_offset;
result = max(result, activation_min);
result = min(result, activation_max);
out_data[out_c] = (int8_t) result;
}
}

72
code/components/esp-nn/src/pooling/esp_nn_avg_pool_ansi.c Normal file
View File

@@ -0,0 +1,72 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
void esp_nn_avg_pool_s8_ansi(const int8_t *input,
const uint16_t input_wd,
const uint16_t input_ht,
int8_t *output,
const uint16_t output_wd,
const uint16_t output_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t filter_wd,
const uint16_t filter_ht,
const uint16_t pad_wd,
const uint16_t pad_ht,
const int32_t activation_min,
const int32_t activation_max,
const uint16_t channels)
{
int32_t base_y = -pad_ht;
for (int32_t out_y = 0; out_y < output_ht; out_y++, base_y += stride_ht) {
int32_t base_x = -pad_wd;
for (int32_t out_x = 0; out_x < output_wd; out_x++, base_x += stride_wd) {
for (int32_t ch_idx = 0; ch_idx < channels; ch_idx++) {
int32_t result = 0;
int32_t filter_cnt = 0;
/* Make sure filter does not cross the input box */
int32_t filter_y_start = max(0, -base_y);
int32_t filter_x_start = max(0, -base_x);
int32_t filter_y_end = min(filter_ht, input_ht - base_y);
int32_t filter_x_end = min(filter_wd, input_wd - base_x);
for (int32_t filter_y = filter_y_start; filter_y < filter_y_end; filter_y++) {
for (int32_t filter_x = filter_x_start; filter_x < filter_x_end; filter_x++) {
int32_t in_x_idx = base_x + filter_x;
int32_t in_y_idx = base_y + filter_y;
int32_t input_index = (in_y_idx * input_wd + in_x_idx) * channels + ch_idx;
result += input[input_index];
filter_cnt++;
}
}
/* Rounded average */
result = result > 0 ? (result + filter_cnt / 2) / filter_cnt
: (result - filter_cnt / 2) / filter_cnt;
/* Activation function */
result = max(result, activation_min);
result = min(result, activation_max);
int32_t output_index = (out_y * output_wd + out_x) * channels + ch_idx;
output[output_index] = (int8_t) result;
}
}
}
}

66
code/components/esp-nn/src/pooling/esp_nn_max_pool_ansi.c Normal file
View File

@@ -0,0 +1,66 @@
// Copyright 2020-2021 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
void esp_nn_max_pool_s8_ansi(const int8_t *input,
const uint16_t input_wd,
const uint16_t input_ht,
int8_t *output,
const uint16_t output_wd,
const uint16_t output_ht,
const uint16_t stride_wd,
const uint16_t stride_ht,
const uint16_t filter_wd,
const uint16_t filter_ht,
const uint16_t pad_wd,
const uint16_t pad_ht,
const int32_t activation_min,
const int32_t activation_max,
const uint16_t channels)
{
int32_t base_y = -pad_ht;
for (int32_t out_y = 0; out_y < output_ht; out_y++, base_y += stride_ht) {
int32_t base_x = -pad_wd;
for (int32_t out_x = 0; out_x < output_wd; out_x++, base_x += stride_wd) {
/* Make sure filter does not cross the input box */
int32_t filter_y_start = max(0, -base_y);
int32_t filter_x_start = max(0, -base_x);
int32_t filter_y_end = min(filter_ht, input_ht - base_y);
int32_t filter_x_end = min(filter_wd, input_wd - base_x);
for (int32_t ch_idx = 0; ch_idx < channels; ch_idx++) {
int8_t result = INT8_MIN;
for (int32_t filter_y = filter_y_start; filter_y < filter_y_end; filter_y++) {
for (int32_t filter_x = filter_x_start; filter_x < filter_x_end; filter_x++) {
int32_t in_x_idx = base_x + filter_x;
int32_t in_y_idx = base_y + filter_y;
int32_t input_index = (in_y_idx * input_wd + in_x_idx) * channels + ch_idx;
result = max(input[input_index], result);
}
}
/* Activation function */
result = max(result, activation_min);
result = min(result, activation_max);
int32_t output_index = (out_y * output_wd + out_x) * channels + ch_idx;
output[output_index] = result;
}
}
}
}

88
code/components/esp-nn/src/softmax/esp_nn_softmax_ansi.c Normal file
View File

@@ -0,0 +1,88 @@
// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "softmax_common.h"
int32_t esp_nn_get_softmax_scratch_size_ansi(const int32_t width, const int32_t height)
{
(void) width;
(void) height;
return 0;
}
void esp_nn_set_softmax_scratch_buf_ansi(void *buffer)
{
(void) buffer;
return;
}
void esp_nn_softmax_s8_ansi(const int8_t *input_data,
const int32_t height,
const int32_t width,
const int32_t mult,
const int32_t shift,
const int32_t diff_min,
int8_t *output_data)
{
// The representation chosen for the input to the exp() function is Q5.26.
// We need to leave extra space since values that we skip might be as large as
// -32 before multiplying by input mult, and therefore as large as
// -16 afterwards. Note that exp(-8) is definitely not insignificant to
// accumulation, but exp(-16) definitely is.
#define ACCUM_BITS 12
#define DIFF_BITS 5
const int32_t mask = (1 << shift);
int32_t col = 0;
const int8_t *in_ptr = input_data;
int8_t *out_ptr = output_data;
for (int row_idx = 0; row_idx < height; row_idx++) {
int8_t max_in_row = in_ptr[0];
for (col = 1; col < width; col++) {
max_in_row = max(max_in_row, in_ptr[col]);
}
int32_t input_diff = 0;
int32_t sum_of_exps = 0;
for (col = 0; col < width; col++) {
input_diff = in_ptr[col] - max_in_row;
if (input_diff >= diff_min) {
const int32_t input_diff_rescaled = SAT_HIGH_MUL(input_diff * mask, mult);
const int32_t exp_raw = esp_nn_exp_on_negative_values(input_diff_rescaled);
sum_of_exps += DIV_POW2(exp_raw, ACCUM_BITS);
}
}
const int32_t headroom_plus1 = esp_nn_clz32((uint32_t) sum_of_exps);
const int32_t shifted_scale = ONE_OVER_ONE_X((sum_of_exps << headroom_plus1) - (1 << 31));
const int32_t bits_over_unit = ACCUM_BITS - headroom_plus1 + 31 - sizeof(int8_t) * 8;
for (col = 0; col < width; col++) {
input_diff = in_ptr[col] - max_in_row;
if (input_diff >= diff_min) {
const int32_t input_diff_rescaled = SAT_HIGH_MUL(input_diff * mask, mult);
const int32_t exp_raw = esp_nn_exp_on_negative_values(input_diff_rescaled);
const int32_t shifted_output = SAT_HIGH_MUL(shifted_scale, exp_raw);
const int32_t result = DIV_POW2(shifted_output, bits_over_unit) - 128;
out_ptr[col] = (int8_t) esp_nn_saturate8(result);
} else {
out_ptr[col] = -128;
}
}
in_ptr += width;
out_ptr += width;
}
}

108
code/components/esp-nn/src/softmax/esp_nn_softmax_opt.c Normal file
View File

@@ -0,0 +1,108 @@
// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "softmax_common.h"
#include <stdio.h>
static int32_t *scratch_buf = NULL;
/**
* @brief Get scratch buffer size needed by softmax function
*
* @param width
* @param height
* @return size in bytes
*
* @note buffer must be 4 byte aligned
*/
int32_t esp_nn_get_softmax_scratch_size_opt(const int32_t width, const int32_t height)
{
(void) height;
return width * 4;
}
/**
* @brief Set scratch buffer to be used by softmax function
*
* @param buffer this can be NULL if one needs to unset it
* must be aligned to 4 bytes
*/
void esp_nn_set_softmax_scratch_buf_opt(void *buffer)
{
scratch_buf = (int32_t *) buffer;
}
void esp_nn_softmax_s8_opt(const int8_t *input_data,
const int32_t height,
const int32_t width,
const int32_t mult,
const int32_t shift,
const int32_t diff_min,
int8_t *output_data)
{
if (scratch_buf == NULL) {
printf("%s error! scratch buffer not set\n", __FUNCTION__);
return;
}
// The representation chosen for the input to the exp() function is Q5.26.
// We need to leave extra space since values that we skip might be as large as
// -32 before multiplying by input mult, and therefore as large as
// -16 afterwards. Note that exp(-8) is definitely not insignificant to
// accumulation, but exp(-16) definitely is.
#define ACCUM_BITS 12
#define DIFF_BITS 5
const int32_t mask = (1 << shift);
int32_t col = 0;
const int8_t *in_ptr = input_data;
int8_t *out_ptr = output_data;
for (int row_idx = 0; row_idx < height; row_idx++) {
int8_t max_in_row = in_ptr[0];
for (col = 1; col < width; col++) {
max_in_row = max(max_in_row, in_ptr[col]);
}
int32_t input_diff = 0;
int32_t sum_of_exps = 0;
for (col = 0; col < width; col++) {
input_diff = in_ptr[col] - max_in_row;
if (input_diff >= diff_min) {
const int32_t input_diff_rescaled = SAT_HIGH_MUL(input_diff * mask, mult);
const int32_t exp_raw = esp_nn_exp_on_negative_values(input_diff_rescaled);
scratch_buf[col] = exp_raw; // store to avoid duplicate calculation later
sum_of_exps += DIV_POW2(exp_raw, ACCUM_BITS);
}
}
const int32_t headroom_plus1 = esp_nn_clz32((uint32_t) sum_of_exps);
const int32_t shifted_scale = ONE_OVER_ONE_X((sum_of_exps << headroom_plus1) - (1 << 31));
const int32_t bits_over_unit = ACCUM_BITS - headroom_plus1 + 31 - sizeof(int8_t) * 8;
for (col = 0; col < width; col++) {
input_diff = in_ptr[col] - max_in_row;
if (input_diff >= diff_min) {
int32_t exp_raw = scratch_buf[col];
const int32_t shifted_output = SAT_HIGH_MUL(shifted_scale, exp_raw);
const int32_t result = DIV_POW2(shifted_output, bits_over_unit) - 128;
out_ptr[col] = (int8_t) esp_nn_saturate8(result);
} else {
out_ptr[col] = -128;
}
}
in_ptr += width;
out_ptr += width;
}
}

104
code/components/esp-nn/src/softmax/softmax_common.h Normal file
View File

@@ -0,0 +1,104 @@
// Copyright 2022 Espressif Systems (Shanghai) PTE LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdint.h>
#include <common_functions.h>
#define MASK_IF_ZERO(x) (x) == 0 ? ~0 : 0
#define MASK_IF_NON_ZERO(x) (x) != 0 ? ~0 : 0
#define SELECT_USING_MASK(mask, a, b) ((mask) & (a)) ^ (~(mask) & (b))
#define SAT_HIGH_MUL(x, y) esp_nn_sat_round_doubling_high_mul((x), (y))
#define DIV_POW2(x,y) esp_nn_div_by_power_of_two((x), (y))
__NN_FORCE_INLINE__ int32_t mul_power_of_2(int val, int exp)
{
const int32_t thresh = ((1 << (31 - exp)) - 1);
int32_t result = val << exp;
result = SELECT_USING_MASK(MASK_IF_NON_ZERO(val > thresh), INT32_MAX, result);
result = SELECT_USING_MASK(MASK_IF_NON_ZERO(val < -thresh), INT32_MIN, result);
return result;
}
/**
* @brief Calculate `1 / (1 + x)` for x in [0, 1]
*
* @param val input value to calculate `1/(1+x)` for
* @return `int32_t` result
* @note Newton-Raphson division
*
* https://en.wikipedia.org/wiki/Division_algorithm#Newton.E2.80.93Raphson_division
* Refer to that page for the logic behind the 48/17 and 32/17 constants.
* Pseudocode: https://en.wikipedia.org/wiki/Division_algorithm#Pseudocode
*/
__NN_FORCE_INLINE__ int32_t esp_nn_one_over_one_plus_x_for_x_in_0_1(int32_t val)
{
const int64_t sum = (int64_t) val + INT32_MAX;
const int32_t half_denominator = (int32_t) ((sum + (sum >= 0 ? 1 : -1)) / 2L);
int32_t constant_48_over_17 = 1515870810;
int32_t constant_neg_32_over_17 = -1010580540;
int32_t x = constant_48_over_17 + SAT_HIGH_MUL(half_denominator, constant_neg_32_over_17);
const int32_t fixed_2_one = (1 << 29);
x += mul_power_of_2(SAT_HIGH_MUL(x, fixed_2_one - SAT_HIGH_MUL(half_denominator, x)), 2);
x += mul_power_of_2(SAT_HIGH_MUL(x, fixed_2_one - SAT_HIGH_MUL(half_denominator, x)), 2);
x += mul_power_of_2(SAT_HIGH_MUL(x, fixed_2_one - SAT_HIGH_MUL(half_denominator, x)), 2);
return mul_power_of_2(x, 1);
}
#define ONE_OVER_ONE_X(x) esp_nn_one_over_one_plus_x_for_x_in_0_1((x))
/**
* @brief Return exp(x) for x < 0.
*
*/
__NN_FORCE_INLINE__ int32_t esp_nn_exp_on_negative_values(int32_t val)
{
int32_t shift = 24;
const int32_t one_quarter = (1 << shift);
int32_t mask = one_quarter - 1;
const int32_t val_mod_minus_quarter = (val & mask) - one_quarter;
const int32_t remainder = val_mod_minus_quarter - val;
// calculate exponent for x in [-1/4, 0) in `result`
const int32_t x = (val_mod_minus_quarter << 5) + (1 << 28);
const int32_t x2 = SAT_HIGH_MUL(x, x);
const int32_t x3 = SAT_HIGH_MUL(x2, x);
const int32_t x4 = SAT_HIGH_MUL(x2, x2);
const int32_t one_over_3 = 715827883;
const int32_t one_over_8 = 1895147668;
const int32_t x4_over_4 = DIV_POW2(x4, 2);
const int32_t x4_over_4_plus_x3_over_6_plus_x2_over_2 = DIV_POW2(SAT_HIGH_MUL(x4_over_4 + x3, one_over_3) + x2, 1);
int32_t result = one_over_8 + SAT_HIGH_MUL(one_over_8, x + x4_over_4_plus_x3_over_6_plus_x2_over_2);
#define SELECT_IF_NON_ZERO(x) { \
mask = MASK_IF_NON_ZERO(remainder & (1 << shift++)); \
result = SELECT_USING_MASK(mask, SAT_HIGH_MUL(result, x), result); \
}
SELECT_IF_NON_ZERO(1672461947)
SELECT_IF_NON_ZERO(1302514674)
SELECT_IF_NON_ZERO(790015084)
SELECT_IF_NON_ZERO(290630308)
SELECT_IF_NON_ZERO(39332535)
SELECT_IF_NON_ZERO(720401)
SELECT_IF_NON_ZERO(242)
#undef SELECT_IF_NON_ZERO
mask = MASK_IF_ZERO(val);
return SELECT_USING_MASK(mask, INT32_MAX, result);
}