diff --git a/measurements/variant_float_inlining_SoA_NEON_polyexp_moreavx_ilp_2_rsqrt_morrsqrt/arm/1.log b/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/1.log similarity index 59% rename from measurements/variant_float_inlining_SoA_NEON_polyexp_moreavx_ilp_2_rsqrt_morrsqrt/arm/1.log rename to measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/1.log index ef9ea2b..a22f38d 100644 --- a/measurements/variant_float_inlining_SoA_NEON_polyexp_moreavx_ilp_2_rsqrt_morrsqrt/arm/1.log +++ b/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/1.log @@ -1,58 +1,66 @@ rm -f main_* run_tests *.o *.s variants/*.s variants/*.o headers/*.s headers/*.o -sudo /usr/bin/g++ main-float.cpp ./variants/variant_float_inlining_SoA_NEON_polyexp_moreavx_ilp_2_rsqrt_morrsqrt.cpp -o main_float_inlining_SoA_NEON_polyexp_moreavx_ilp_2_rsqrt_morrsqrt -O3 -march=native -std=c++23 -Wall -Wextra -flto -DFLOPCOUNT=0 -DRANDOM_SEED=0 -DROOFLINE=0 -DBENCHMARK=1 +sudo /usr/bin/g++ main-float.cpp ./variants/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix.cpp -o main_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix -O3 -march=native -std=c++23 -Wall -Wextra -flto -DFLOPCOUNT=0 -DRANDOM_SEED=0 -DROOFLINE=0 -DBENCHMARK=1 === N_P=50 N_ITER=200 === W(n) per iteration: 183700.00 flops Q(n) per iteration: 3256 bytes I(n): 56.42 flops/byte W(n): 36740000.00 flops -T(n): 4492093.50 cycles -P(n): 8.18 flops/cycle +T(n): 5891612.50 cycles +P(n): 6.24 flops/cycle === N_P=100 N_ITER=200 === W(n) per iteration: 727400.00 flops Q(n) per iteration: 6456 bytes I(n): 112.67 flops/byte W(n): 145480000.00 flops -T(n): 15717616.00 cycles -P(n): 9.26 flops/cycle +T(n): 20903770.00 cycles +P(n): 6.96 flops/cycle === N_P=200 N_ITER=200 === W(n) per iteration: 2894800.00 flops Q(n) per iteration: 12856 bytes I(n): 225.17 flops/byte W(n): 578960000.00 flops -T(n): 58648184.00 cycles -P(n): 9.87 flops/cycle +T(n): 78269008.00 cycles +P(n): 7.40 flops/cycle === N_P=500 N_ITER=200 === W(n) per iteration: 18037000.00 flops Q(n) per iteration: 32056 bytes I(n): 562.67 flops/byte W(n): 3607399936.00 flops -T(n): 359248768.00 cycles -P(n): 10.04 flops/cycle +T(n): 484667776.00 cycles +P(n): 7.44 flops/cycle === N_P=1000 N_ITER=200 === W(n) per iteration: 72074000.00 flops Q(n) per iteration: 64056 bytes I(n): 1125.17 flops/byte W(n): 14414799872.00 flops -T(n): 1415209856.00 cycles -P(n): 10.19 flops/cycle +T(n): 1917664384.00 cycles +P(n): 7.52 flops/cycle === N_P=2000 N_ITER=200 === W(n) per iteration: 288148000.00 flops Q(n) per iteration: 128056 bytes I(n): 2250.17 flops/byte W(n): 57629601792.00 flops -T(n): 5657639936.00 cycles -P(n): 10.19 flops/cycle +T(n): 7683078656.00 cycles +P(n): 7.50 flops/cycle === N_P=5000 N_ITER=200 === W(n) per iteration: 1800369920.00 flops Q(n) per iteration: 320056 bytes I(n): 5625.17 flops/byte W(n): 360073986048.00 flops -T(n): 35147825152.00 cycles -P(n): 10.24 flops/cycle +T(n): 48221761536.00 cycles +P(n): 7.47 flops/cycle + +=== N_P=10000 N_ITER=200 === +W(n) per iteration: 7200739840.00 flops +Q(n) per iteration: 640056 bytes +I(n): 11250.17 flops/byte +W(n): 1440147963904.00 flops +T(n): 222472847360.00 cycles +P(n): 6.47 flops/cycle diff --git a/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/2.log b/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/2.log new file mode 100644 index 0000000..a84bca2 --- /dev/null +++ b/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/2.log @@ -0,0 +1,66 @@ +rm -f main_* run_tests *.o *.s variants/*.s variants/*.o headers/*.s headers/*.o +sudo /usr/bin/g++ main-float.cpp ./variants/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix.cpp -o main_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix -O3 -march=native -std=c++23 -Wall -Wextra -flto -DFLOPCOUNT=0 -DRANDOM_SEED=0 -DROOFLINE=0 -DBENCHMARK=1 +=== N_P=50 N_ITER=200 === +W(n) per iteration: 183700.00 flops +Q(n) per iteration: 3256 bytes +I(n): 56.42 flops/byte +W(n): 36740000.00 flops +T(n): 5881106.00 cycles +P(n): 6.25 flops/cycle + +=== N_P=100 N_ITER=200 === +W(n) per iteration: 727400.00 flops +Q(n) per iteration: 6456 bytes +I(n): 112.67 flops/byte +W(n): 145480000.00 flops +T(n): 20868150.00 cycles +P(n): 6.97 flops/cycle + +=== N_P=200 N_ITER=200 === +W(n) per iteration: 2894800.00 flops +Q(n) per iteration: 12856 bytes +I(n): 225.17 flops/byte +W(n): 578960000.00 flops +T(n): 78350920.00 cycles +P(n): 7.39 flops/cycle + +=== N_P=500 N_ITER=200 === +W(n) per iteration: 18037000.00 flops +Q(n) per iteration: 32056 bytes +I(n): 562.67 flops/byte +W(n): 3607399936.00 flops +T(n): 484657088.00 cycles +P(n): 7.44 flops/cycle + +=== N_P=1000 N_ITER=200 === +W(n) per iteration: 72074000.00 flops +Q(n) per iteration: 64056 bytes +I(n): 1125.17 flops/byte +W(n): 14414799872.00 flops +T(n): 1920107648.00 cycles +P(n): 7.51 flops/cycle + +=== N_P=2000 N_ITER=200 === +W(n) per iteration: 288148000.00 flops +Q(n) per iteration: 128056 bytes +I(n): 2250.17 flops/byte +W(n): 57629601792.00 flops +T(n): 7641970176.00 cycles +P(n): 7.54 flops/cycle + +=== N_P=5000 N_ITER=200 === +W(n) per iteration: 1800369920.00 flops +Q(n) per iteration: 320056 bytes +I(n): 5625.17 flops/byte +W(n): 360073986048.00 flops +T(n): 47656402944.00 cycles +P(n): 7.56 flops/cycle + +=== N_P=10000 N_ITER=200 === +W(n) per iteration: 7200739840.00 flops +Q(n) per iteration: 640056 bytes +I(n): 11250.17 flops/byte +W(n): 1440147963904.00 flops +T(n): 221504274432.00 cycles +P(n): 6.50 flops/cycle + diff --git a/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/3.log b/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/3.log new file mode 100644 index 0000000..f9d2679 --- /dev/null +++ b/measurements/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix/arm/3.log @@ -0,0 +1,66 @@ +rm -f main_* run_tests *.o *.s variants/*.s variants/*.o headers/*.s headers/*.o +sudo /usr/bin/g++ main-float.cpp ./variants/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix.cpp -o main_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix -O3 -march=native -std=c++23 -Wall -Wextra -flto -DFLOPCOUNT=0 -DRANDOM_SEED=0 -DROOFLINE=0 -DBENCHMARK=1 +=== N_P=50 N_ITER=200 === +W(n) per iteration: 183700.00 flops +Q(n) per iteration: 3256 bytes +I(n): 56.42 flops/byte +W(n): 36740000.00 flops +T(n): 5887339.50 cycles +P(n): 6.24 flops/cycle + +=== N_P=100 N_ITER=200 === +W(n) per iteration: 727400.00 flops +Q(n) per iteration: 6456 bytes +I(n): 112.67 flops/byte +W(n): 145480000.00 flops +T(n): 20857236.00 cycles +P(n): 6.98 flops/cycle + +=== N_P=200 N_ITER=200 === +W(n) per iteration: 2894800.00 flops +Q(n) per iteration: 12856 bytes +I(n): 225.17 flops/byte +W(n): 578960000.00 flops +T(n): 78339488.00 cycles +P(n): 7.39 flops/cycle + +=== N_P=500 N_ITER=200 === +W(n) per iteration: 18037000.00 flops +Q(n) per iteration: 32056 bytes +I(n): 562.67 flops/byte +W(n): 3607399936.00 flops +T(n): 483592896.00 cycles +P(n): 7.46 flops/cycle + +=== N_P=1000 N_ITER=200 === +W(n) per iteration: 72074000.00 flops +Q(n) per iteration: 64056 bytes +I(n): 1125.17 flops/byte +W(n): 14414799872.00 flops +T(n): 1918282240.00 cycles +P(n): 7.51 flops/cycle + +=== N_P=2000 N_ITER=200 === +W(n) per iteration: 288148000.00 flops +Q(n) per iteration: 128056 bytes +I(n): 2250.17 flops/byte +W(n): 57629601792.00 flops +T(n): 7707683840.00 cycles +P(n): 7.48 flops/cycle + +=== N_P=5000 N_ITER=200 === +W(n) per iteration: 1800369920.00 flops +Q(n) per iteration: 320056 bytes +I(n): 5625.17 flops/byte +W(n): 360073986048.00 flops +T(n): 47889321984.00 cycles +P(n): 7.52 flops/cycle + +=== N_P=10000 N_ITER=200 === +W(n) per iteration: 7200739840.00 flops +Q(n) per iteration: 640056 bytes +I(n): 11250.17 flops/byte +W(n): 1440147963904.00 flops +T(n): 221847699456.00 cycles +P(n): 6.49 flops/cycle + diff --git a/variants/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix.cpp b/variants/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix.cpp new file mode 100644 index 0000000..14ce633 --- /dev/null +++ b/variants/variant_float_inlining_SoA_NEON_polyexp_morevect_ilp_2_rsqrt_morersqrt_expfix.cpp @@ -0,0 +1,648 @@ +// # Changes: NEON port of variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_rsqrt.cpp +// - 16-wide AVX-512 → 4-wide NEON; ILP unroll by 2 preserved (inner loop processes 8 j's per iter) +// - rsqrt14_ps → vrsqrteq_f32 + 1 Newton iteration (single-precision accuracy) +// - reduce_add_ps → vaddvq_f32 +// - Padding multiple changed from 32 to 8 +// - Scalar obstacle loop unchanged + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#ifdef __aarch64__ +#include +#endif + +#ifdef __x86_64__ +#define SIMDE_ENABLE_NATIVE_ALIASES +#include +#endif + +#include +#include +#include +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 7) & ~7; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +// # 64 MULTs, 56 ADDs +static inline float32x4_t _neon_exp_ps(float32x4_t x) { + const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); + + // # 4 MULTS + float32x4_t nf = vrndnq_f32(vmulq_f32(x, log2e)); + + const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); + + // # 4 MULTs, 4 ADDs + float32x4_t r = vfmsq_f32(x, nf, ln2); // r = x - nf*ln2 + + const float32x4_t c12 = vdupq_n_f32(c12_scalar); + const float32x4_t c11 = vdupq_n_f32(c11_scalar); + const float32x4_t c10 = vdupq_n_f32(c10_scalar); + const float32x4_t c9 = vdupq_n_f32(c9_scalar); + const float32x4_t c8 = vdupq_n_f32(c8_scalar); + const float32x4_t c7 = vdupq_n_f32(c7_scalar); + const float32x4_t c6 = vdupq_n_f32(c6_scalar); + const float32x4_t c5 = vdupq_n_f32(c5_scalar); + const float32x4_t c4 = vdupq_n_f32(c4_scalar); + const float32x4_t c3 = vdupq_n_f32(c3_scalar); + const float32x4_t c2 = vdupq_n_f32(c2_scalar); + const float32x4_t c1 = vdupq_n_f32(c1_scalar); + const float32x4_t c0 = vdupq_n_f32(c0_scalar); + + float32x4_t p = c12; + + // # 13 * (4 MULTs, 4 ADDs) + // # = 52 MULTs, 52 ADDs + p = vfmaq_f32(c12, p, r); + p = vfmaq_f32(c11, p, r); + p = vfmaq_f32(c10, p, r); + p = vfmaq_f32(c9, p, r); + p = vfmaq_f32(c8, p, r); + p = vfmaq_f32(c7, p, r); + p = vfmaq_f32(c6, p, r); + p = vfmaq_f32(c5, p, r); + p = vfmaq_f32(c4, p, r); + p = vfmaq_f32(c3, p, r); + p = vfmaq_f32(c2, p, r); + p = vfmaq_f32(c1, p, r); + p = vfmaq_f32(c0, p, r); + + int32x4_t ni = vcvtq_s32_f32(nf); + int32x4_t biased = vaddq_s32(ni, vdupq_n_s32(127)); + int32x4_t scaled = vshlq_n_s32(biased, 23); + float32x4_t pow2n = vreinterpretq_f32_s32(scaled); + + // # 4 MULTs + return vmulq_f32(p, pow2n); +} + +// # 4 ADD, 16 MULTS, 4 SQRT, 4 DIV +// Newton-Raphson refinement formula. +static inline float32x4_t _neon_rsqrt_ps(float32x4_t x) { + // # 4 SQRT, 4 DIV + float32x4_t y = vrsqrteq_f32(x); + + // # 16 MULTS, 4 ADD + y = vmulq_f32(y, vrsqrtsq_f32(vmulq_f32(x, y), y)); + return y; +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 7) & ~7; + + const float32x4_t lane_off = {0.0f, 1.0f, 2.0f, 3.0f}; + + const float32x4_t v_neg_v0_div_2s = vdupq_n_f32(NEG_V0_AB_div_TWO_SIGMA); + const float32x4_t v_neg_2s_inv = vdupq_n_f32(NEG_TWO_SIGMA_INV); + const float32x4_t v_cos_phi_sq = vdupq_n_f32(cos_phi_sq); + const float32x4_t v_dt_neg = vdupq_n_f32(dt_neg); + const float32x4_t v_c_neg_dt = vdupq_n_f32(c_neg_times_dt); + const float32x4_t v_zero = vdupq_n_f32(0.0f); + const float32x4_t v_one = vdupq_n_f32(1.0f); + const float32x4_t v_eps_sq = vdupq_n_f32(EPS_SQ); + const float32x4_t v_np = vdupq_n_f32((float)n_p); + const float32x4_t v_exp_min = vdupq_n_f32(-87.0f); + + // # ADDs: n_iter * ( + // # n_p * 20 + + // # n_p * n_p_pad * 32 + + // # n_p * board.n_obstacles * 10 + + // # n_p_pad * 9 + // # ) + + // # MULTs: n_iter * ( + // # n_p * 4 + + // # n_p * n_p_pad * 54 + + // # n_p * board.n_obstacles * 13 + + // # n_p_pad * 24 + // # ) + + // # DIVs: n_iter * ( + // # n_p * n_p_pad * 3 + + // # n_p * board.n_obstacles + + // # n_p_pad * 2 + // # ) + + // # SQRTs: n_iter * ( + // # n_p * n_p_pad * 3 + + // # n_p * board.n_obstacles + + // # n_p_pad * 3 + // # ) + + // # EXPs: n_iter * n_p * board.n_obstacles + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + + // # 2 MULTS + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + // # 2 ADDS, 2 MULTS + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const float32x4_t pix = vdupq_n_f32(pos_x[i]); + const float32x4_t piy = vdupq_n_f32(pos_y[i]); + const float32x4_t eix = vdupq_n_f32(e_x[i]); + const float32x4_t eiy = vdupq_n_f32(e_y[i]); + const float32x4_t v_i = vdupq_n_f32((float)i); + + float32x4_t acc_x_0 = vdupq_n_f32(0.0f); + float32x4_t acc_y_0 = vdupq_n_f32(0.0f); + float32x4_t acc_x_1 = vdupq_n_f32(0.0f); + float32x4_t acc_y_1 = vdupq_n_f32(0.0f); + + for (int j = 0; j < n_p_pad; j += 8) { + + // # 8 ADDS + float32x4_t j_idx_0 = vaddq_f32(vdupq_n_f32((float)j), lane_off); + float32x4_t j_idx_1 = vaddq_f32(vdupq_n_f32((float)(j + 4)), lane_off); + + uint32x4_t m_neq_i_0 = vmvnq_u32(vceqq_f32(j_idx_0, v_i)); + uint32x4_t m_neq_i_1 = vmvnq_u32(vceqq_f32(j_idx_1, v_i)); + uint32x4_t m_in_range_0 = vcltq_f32(j_idx_0, v_np); + uint32x4_t m_in_range_1 = vcltq_f32(j_idx_1, v_np); + uint32x4_t active_0 = vandq_u32(m_neq_i_0, m_in_range_0); + uint32x4_t active_1 = vandq_u32(m_neq_i_1, m_in_range_1); + + float32x4_t pjx_0 = vld1q_f32(&pos_x[j]); + float32x4_t pjx_1 = vld1q_f32(&pos_x[j + 4]); + float32x4_t pjy_0 = vld1q_f32(&pos_y[j]); + float32x4_t pjy_1 = vld1q_f32(&pos_y[j + 4]); + float32x4_t sbe_jx_0 = vld1q_f32(&sb_e_x[j]); + float32x4_t sbe_jx_1 = vld1q_f32(&sb_e_x[j + 4]); + float32x4_t sbe_jy_0 = vld1q_f32(&sb_e_y[j]); + float32x4_t sbe_jy_1 = vld1q_f32(&sb_e_y[j + 4]); + + // We negate the values to be able to perform an fma when computing two_b below + float32x4_t sb_jsq_0_neg = vnegq_f32(vld1q_f32(&sb_sq[j])); + float32x4_t sb_jsq_1_neg = vnegq_f32(vld1q_f32(&sb_sq[j + 4])); + + // # 16 ADDS + float32x4_t r_ab_x_0 = vsubq_f32(pix, pjx_0); + float32x4_t r_ab_x_1 = vsubq_f32(pix, pjx_1); + float32x4_t r_ab_y_0 = vsubq_f32(piy, pjy_0); + float32x4_t r_ab_y_1 = vsubq_f32(piy, pjy_1); + + // # 8 ADDS, 16 MULTS + float32x4_t r_ab_n2_0 = vfmaq_f32(vmulq_f32(r_ab_y_0, r_ab_y_0), r_ab_x_0, r_ab_x_0); + float32x4_t r_ab_n2_1 = vfmaq_f32(vmulq_f32(r_ab_y_1, r_ab_y_1), r_ab_x_1, r_ab_x_1); + + r_ab_n2_0 = vmaxq_f32(r_ab_n2_0, v_eps_sq); + r_ab_n2_1 = vmaxq_f32(r_ab_n2_1, v_eps_sq); + + // # 8 ADD, 32 MULTS, 8 SQRT, 8 DIV + float32x4_t inv_r_ab_n_0 = _neon_rsqrt_ps(r_ab_n2_0); + float32x4_t inv_r_ab_n_1 = _neon_rsqrt_ps(r_ab_n2_1); + + // # 8 MULTS + float32x4_t r_ab_n_0 = vmulq_f32(r_ab_n2_0, inv_r_ab_n_0); + float32x4_t r_ab_n_1 = vmulq_f32(r_ab_n2_1, inv_r_ab_n_1); + + // # 16 ADDS + float32x4_t r_ab2_x_0 = vsubq_f32(r_ab_x_0, sbe_jx_0); + float32x4_t r_ab2_x_1 = vsubq_f32(r_ab_x_1, sbe_jx_1); + float32x4_t r_ab2_y_0 = vsubq_f32(r_ab_y_0, sbe_jy_0); + float32x4_t r_ab2_y_1 = vsubq_f32(r_ab_y_1, sbe_jy_1); + + // # 8 ADDS, 16 MULTS + float32x4_t r_ab2_n2_0 = vfmaq_f32(vmulq_f32(r_ab2_y_0, r_ab2_y_0), r_ab2_x_0, r_ab2_x_0); + float32x4_t r_ab2_n2_1 = vfmaq_f32(vmulq_f32(r_ab2_y_1, r_ab2_y_1), r_ab2_x_1, r_ab2_x_1); + + r_ab2_n2_0 = vmaxq_f32(r_ab2_n2_0, v_eps_sq); + r_ab2_n2_1 = vmaxq_f32(r_ab2_n2_1, v_eps_sq); + + // # 8 ADD, 32 MULTS, 8 SQRT, 8 DIV + float32x4_t inv_r_ab2_n_0 = _neon_rsqrt_ps(r_ab2_n2_0); + float32x4_t inv_r_ab2_n_1 = _neon_rsqrt_ps(r_ab2_n2_1); + + // # 8 MULTS + float32x4_t r_ab2_n_0 = vmulq_f32(r_ab2_n2_0, inv_r_ab2_n_0); + float32x4_t r_ab2_n_1 = vmulq_f32(r_ab2_n2_1, inv_r_ab2_n_1); + + // # 8 ADDS + float32x4_t r_sum_0 = vaddq_f32(r_ab_n_0, r_ab2_n_0); + float32x4_t r_sum_1 = vaddq_f32(r_ab_n_1, r_ab2_n_1); + + // # 8 ADDS, 8 MULTS + float32x4_t two_b_sq_0 = vfmaq_f32(sb_jsq_0_neg, r_sum_0, r_sum_0); + float32x4_t two_b_sq_1 = vfmaq_f32(sb_jsq_1_neg, r_sum_1, r_sum_1); + + two_b_sq_0 = vmaxq_f32(two_b_sq_0, v_eps_sq); + two_b_sq_1 = vmaxq_f32(two_b_sq_1, v_eps_sq); + + // # 8 ADD, 32 MULTS, 8 SQRT, 8 DIV + float32x4_t inv_two_b_0 = _neon_rsqrt_ps(two_b_sq_0); + float32x4_t inv_two_b_1 = _neon_rsqrt_ps(two_b_sq_1); + + // # 8 MULTS + float32x4_t two_b_0 = vmulq_f32(two_b_sq_0, inv_two_b_0); + float32x4_t two_b_1 = vmulq_f32(two_b_sq_1, inv_two_b_1); + + // # 8 MULTS + float32x4_t e_arg_0 = vmulq_f32(two_b_0, v_neg_2s_inv); + float32x4_t e_arg_1 = vmulq_f32(two_b_1, v_neg_2s_inv); + + e_arg_0 = vmaxq_f32(e_arg_0, v_exp_min); + e_arg_1 = vmaxq_f32(e_arg_1, v_exp_min); + + // # 128 MULTs, 112 ADDs + float32x4_t e_val_0 = _neon_exp_ps(e_arg_0); + float32x4_t e_val_1 = _neon_exp_ps(e_arg_1); + + // # 24 MULTS + float32x4_t numer_inv_two_b_0 = vmulq_f32(vmulq_f32(r_sum_0, v_neg_v0_div_2s), vmulq_f32(e_val_0, inv_two_b_0)); + float32x4_t numer_inv_two_b_1 = vmulq_f32(vmulq_f32(r_sum_1, v_neg_v0_div_2s), vmulq_f32(e_val_1, inv_two_b_1)); + + // # 16 MULTS + float32x4_t sm_r_ab_0 = vmulq_f32(numer_inv_two_b_0, inv_r_ab_n_0); + float32x4_t sm_r_ab_1 = vmulq_f32(numer_inv_two_b_1, inv_r_ab_n_1); + float32x4_t sm_r_ab2_0 = vmulq_f32(numer_inv_two_b_0, inv_r_ab2_n_0); + float32x4_t sm_r_ab2_1 = vmulq_f32(numer_inv_two_b_1, inv_r_ab2_n_1); + + // # 16 ADDS, 32 MULTS + float32x4_t f_x_0 = vfmaq_f32(vmulq_f32(r_ab_x_0, sm_r_ab_0), r_ab2_x_0, sm_r_ab2_0); + float32x4_t f_x_1 = vfmaq_f32(vmulq_f32(r_ab_x_1, sm_r_ab_1), r_ab2_x_1, sm_r_ab2_1); + float32x4_t f_y_0 = vfmaq_f32(vmulq_f32(r_ab_y_0, sm_r_ab_0), r_ab2_y_0, sm_r_ab2_0); + float32x4_t f_y_1 = vfmaq_f32(vmulq_f32(r_ab_y_1, sm_r_ab_1), r_ab2_y_1, sm_r_ab2_1); + + // # 8 ADDS, 16 MULTS + float32x4_t f_norm_sq_0 = vfmaq_f32(vmulq_f32(f_y_0, f_y_0), f_x_0, f_x_0); + float32x4_t f_norm_sq_1 = vfmaq_f32(vmulq_f32(f_y_1, f_y_1), f_x_1, f_x_1); + + // # 8 MULTS + float32x4_t rhs_0 = vmulq_f32(f_norm_sq_0, v_cos_phi_sq); + float32x4_t rhs_1 = vmulq_f32(f_norm_sq_1, v_cos_phi_sq); + + // # 8 ADDS, 16 MULTS + float32x4_t e_dot_f_0 = vfmaq_f32(vmulq_f32(eiy, f_y_0), eix, f_x_0); + float32x4_t e_dot_f_1 = vfmaq_f32(vmulq_f32(eiy, f_y_1), eix, f_x_1); + + // # 8 MULTS + float32x4_t e_dot_f_sq_0 = vmulq_f32(e_dot_f_0, e_dot_f_0); + float32x4_t e_dot_f_sq_1 = vmulq_f32(e_dot_f_1, e_dot_f_1); + + uint32x4_t c1_0 = vcltq_f32(e_dot_f_0, v_zero); + uint32x4_t c1_1 = vcltq_f32(e_dot_f_1, v_zero); + uint32x4_t c2_0 = vcgtq_f32(e_dot_f_sq_0, rhs_0); + uint32x4_t c2_1 = vcgtq_f32(e_dot_f_sq_1, rhs_1); + + uint32x4_t cond_0 = vandq_u32(c1_0, c2_0); + uint32x4_t cond_1 = vandq_u32(c1_1, c2_1); + + float32x4_t w_val_0 = vbslq_f32(cond_0, v_c_neg_dt, v_dt_neg); + float32x4_t w_val_1 = vbslq_f32(cond_1, v_c_neg_dt, v_dt_neg); + + float32x4_t f_x_safe_0 = vbslq_f32(active_0, f_x_0, v_zero); + float32x4_t f_x_safe_1 = vbslq_f32(active_1, f_x_1, v_zero); + float32x4_t f_y_safe_0 = vbslq_f32(active_0, f_y_0, v_zero); + float32x4_t f_y_safe_1 = vbslq_f32(active_1, f_y_1, v_zero); + + // # 16 ADDS, 16 MULTS + acc_x_0 = vfmaq_f32(acc_x_0, f_x_safe_0, w_val_0); + acc_x_1 = vfmaq_f32(acc_x_1, f_x_safe_1, w_val_1); + acc_y_0 = vfmaq_f32(acc_y_0, f_y_safe_0, w_val_0); + acc_y_1 = vfmaq_f32(acc_y_1, f_y_safe_1, w_val_1); + } + + // # 16 ADDS + f_alpha_dt_x += vaddvq_f32(vaddq_f32(acc_x_0, acc_x_1)); + f_alpha_dt_y += vaddvq_f32(vaddq_f32(acc_y_0, acc_y_1)); + + for (int k = 0; k < board.n_obstacles; k++) { + + // # 2 ADDs + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + // # 3 MULTs, 1 ADD + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + // # 4 ADDs, 2 MULTs + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + // # 1 SQRT, 2 MULTs, 1 ADD + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + // # 4 MULTs, 1 EXP, 1 DIV + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + // # 2 MULTS + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + // # 2 ADDs + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + // # 2 ADDs + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 4) { + + float32x4_t w_x_i_vec = vld1q_f32(&w_x[i]); + float32x4_t w_y_i_vec = vld1q_f32(&w_y[i]); + float32x4_t v_max_i_vec = vld1q_f32(&v_max[i]); + float32x4_t pos_x_i_vec = vld1q_f32(&pos_x[i]); + float32x4_t pos_y_i_vec = vld1q_f32(&pos_y[i]); + float32x4_t destination_y_i_vec = vld1q_f32(&destination_y[i]); + float32x4_t destination_x_i_vec = vld1q_f32(&destination_x[i]); + float32x4_t desired_v0_i_vec = vld1q_f32(&desired_v0[i]); + int32x4_t color_i_vec = vld1q_s32(&color[i]); + + // # 4 ADDS, 8 MULTS + float32x4_t w_norm_sq = vfmaq_f32(vmulq_f32(w_y_i_vec, w_y_i_vec), w_x_i_vec, w_x_i_vec); + + // # 4 ADD, 16 MULTS, 4 SQRT, 4 DIV + float32x4_t w_norm_inv = _neon_rsqrt_ps(w_norm_sq); + + // # 4 MULTS + float32x4_t correction_factor = vminq_f32(v_one, vmulq_f32(v_max_i_vec, w_norm_inv)); + + // # 8 MULTS + float32x4_t curr_vel_x_i_vec = vmulq_f32(w_x_i_vec, correction_factor); + float32x4_t curr_vel_y_i_vec = vmulq_f32(w_y_i_vec, correction_factor); + + const float32x4_t dt_vec = vdupq_n_f32(dt); + + // # 8 ADDS, 8 MULTS + pos_x_i_vec = vfmaq_f32(pos_x_i_vec, curr_vel_x_i_vec, dt_vec); + pos_y_i_vec = vfmaq_f32(pos_y_i_vec, curr_vel_y_i_vec, dt_vec); + + uint32x4_t cond_1_left = vceqq_s32(color_i_vec, vdupq_n_s32(0)); + uint32x4_t cond_1_right = vcgtq_f32(pos_x_i_vec, vdupq_n_f32(BOARD_LENGTH - 1.5f)); + uint32x4_t cond_1 = vandq_u32(cond_1_left, cond_1_right); + + pos_x_i_vec = vbslq_f32(cond_1, v_one, pos_x_i_vec); + destination_y_i_vec = vbslq_f32(cond_1, pos_y_i_vec, destination_y_i_vec); + w_x_i_vec = vbslq_f32(cond_1, desired_v0_i_vec, w_x_i_vec); + w_y_i_vec = vbslq_f32(cond_1, v_zero, w_y_i_vec); + curr_vel_x_i_vec = vbslq_f32(cond_1, w_x_i_vec, curr_vel_x_i_vec); + curr_vel_y_i_vec = vbslq_f32(cond_1, w_y_i_vec, curr_vel_y_i_vec); + + uint32x4_t cond_2_left = vceqq_s32(color_i_vec, vdupq_n_s32(1)); + uint32x4_t cond_2_right = vcltq_f32(pos_x_i_vec, vdupq_n_f32(1.5f)); + uint32x4_t cond_2 = vandq_u32(cond_2_left, cond_2_right); + + pos_x_i_vec = vbslq_f32(cond_2, vdupq_n_f32(BOARD_LENGTH - 1.0f), pos_x_i_vec); + destination_y_i_vec = vbslq_f32(cond_2, pos_y_i_vec, destination_y_i_vec); + + float32x4_t neg_desired_v0 = vnegq_f32(desired_v0_i_vec); + w_x_i_vec = vbslq_f32(cond_2, neg_desired_v0, w_x_i_vec); + w_y_i_vec = vbslq_f32(cond_2, v_zero, w_y_i_vec); + curr_vel_x_i_vec = vbslq_f32(cond_2, w_x_i_vec, curr_vel_x_i_vec); + curr_vel_y_i_vec = vbslq_f32(cond_2, w_y_i_vec, curr_vel_y_i_vec); + + pos_y_i_vec = vmaxq_f32(pos_y_i_vec, vdupq_n_f32(WALL_EPS)); + pos_y_i_vec = vminq_f32(pos_y_i_vec, vdupq_n_f32(BOARD_WIDTH_MINUS_WALL_EPS)); + + // # 8 ADDS + float32x4_t ex_vec = vsubq_f32(destination_x_i_vec, pos_x_i_vec); + float32x4_t ey_vec = vsubq_f32(destination_y_i_vec, pos_y_i_vec); + + // # 4 ADDS, 8 MULTS + float32x4_t e_norm_sq = vfmaq_f32(vmulq_f32(ey_vec, ey_vec), ex_vec, ex_vec); + + // # 4 ADD, 16 MULTS, 4 SQRT, 4 DIV + float32x4_t e_norm_inv = _neon_rsqrt_ps(e_norm_sq); + + // # 8 MULTS + float32x4_t e_x_i_vec = vmulq_f32(ex_vec, e_norm_inv); + float32x4_t e_y_i_vec = vmulq_f32(ey_vec, e_norm_inv); + + // # 4 ADDS, 8 MULTS + float32x4_t vel_norm_sq = vfmaq_f32(vmulq_f32(curr_vel_y_i_vec, curr_vel_y_i_vec), curr_vel_x_i_vec, curr_vel_x_i_vec); + + // # 4 MULTS + float32x4_t sb_sq_i_vec = vmulq_f32(vel_norm_sq, vdupq_n_f32(delta_t_sq)); + + // # 4 SQRT + float32x4_t vel_norm_dt = vsqrtq_f32(sb_sq_i_vec); // full-precision sqrt, matches AVX-512 + + vst1q_f32(&curr_vel_x[i], curr_vel_x_i_vec); + vst1q_f32(&curr_vel_y[i], curr_vel_y_i_vec); + vst1q_f32(&w_x[i], w_x_i_vec); + vst1q_f32(&w_y[i], w_y_i_vec); + vst1q_f32(&pos_x[i], pos_x_i_vec); + vst1q_f32(&pos_y[i], pos_y_i_vec); + vst1q_f32(&e_x[i], e_x_i_vec); + vst1q_f32(&e_y[i], e_y_i_vec); + vst1q_f32(&destination_y[i], destination_y_i_vec); + vst1q_f32(&sb_sq[i], sb_sq_i_vec); + + // # 8 MULTS + vst1q_f32(&sb_e_x[i], vmulq_f32(e_x_i_vec, vel_norm_dt)); + vst1q_f32(&sb_e_y[i], vmulq_f32(e_y_i_vec, vel_norm_dt)); + } + +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +}