diff --git a/Makefile b/Makefile index 653eb89..c939545 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,13 @@ CXXFLAGS += -lsleef endif endif +# sleef is only needed for the avx512 variant (used by SLEEF intrinsics there) +ifeq ($(ARCH),x86_64) +ifneq ($(filter avx512_%,$(VARIANT)),) +CXXFLAGS += -lsleef +endif +endif + VARIANT_SRC := ./variants/variant_$(VARIANT).cpp ifneq ($(filter float_%,$(VARIANT)),) diff --git a/headers/constants.hpp b/headers/constants.hpp index 129d02f..ce3c51f 100644 --- a/headers/constants.hpp +++ b/headers/constants.hpp @@ -114,7 +114,6 @@ const __m512d dt_neg_vec = _mm512_set1_pd(dt_neg); const double CUTOFF = 12.0; const double CUTOFF_SQ = CUTOFF * CUTOFF; - const double cos_phi_sq = cos_phi * cos_phi; const size_t CACHE_BLOCK_SIZE = std::hardware_destructive_interference_size; diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp index 4f6d471..95dd277 100644 --- a/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp @@ -249,11 +249,11 @@ int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, c __mmask16 m_in_range = _mm512_cmp_ps_mask(j_idx, v_np, _CMP_LT_OQ); __mmask16 active = m_neq_i & m_in_range; - __m512 pjx = _mm512_load_ps(&pos_x[j]); - __m512 pjy = _mm512_load_ps(&pos_y[j]); - __m512 sbe_jx = _mm512_load_ps(&sb_e_x[j]); - __m512 sbe_jy = _mm512_load_ps(&sb_e_y[j]); - __m512 sb_jsq = _mm512_load_ps(&sb_sq[j]); + __m512 pjx = _mm512_loadu_ps(&pos_x[j]); + __m512 pjy = _mm512_loadu_ps(&pos_y[j]); + __m512 sbe_jx = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jy = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sb_jsq = _mm512_loadu_ps(&sb_sq[j]); __m512 r_ab_x = _mm512_sub_ps(pix, pjx); __m512 r_ab_y = _mm512_sub_ps(piy, pjy); diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp index d3fcc84..1e3dbb5 100644 --- a/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp @@ -265,11 +265,11 @@ int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, c __mmask16 m_in_range = _mm512_cmp_ps_mask(j_idx, v_np, _CMP_LT_OQ); __mmask16 active = m_neq_i & m_in_range; - __m512 pjx = _mm512_load_ps(&pos_x[j]); - __m512 pjy = _mm512_load_ps(&pos_y[j]); - __m512 sbe_jx = _mm512_load_ps(&sb_e_x[j]); - __m512 sbe_jy = _mm512_load_ps(&sb_e_y[j]); - __m512 sb_jsq = _mm512_load_ps(&sb_sq[j]); + __m512 pjx = _mm512_loadu_ps(&pos_x[j]); + __m512 pjy = _mm512_loadu_ps(&pos_y[j]); + __m512 sbe_jx = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jy = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sb_jsq = _mm512_loadu_ps(&sb_sq[j]); __m512 r_ab_x = _mm512_sub_ps(pix, pjx); __m512 r_ab_y = _mm512_sub_ps(piy, pjy); diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp index 28cb9b2..15e0cbe 100644 --- a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp @@ -249,11 +249,11 @@ int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, c __mmask16 m_in_range = _mm512_cmp_ps_mask(j_idx, v_np, _CMP_LT_OQ); __mmask16 active = m_neq_i & m_in_range; - __m512 pjx = _mm512_load_ps(&pos_x[j]); - __m512 pjy = _mm512_load_ps(&pos_y[j]); - __m512 sbe_jx = _mm512_load_ps(&sb_e_x[j]); - __m512 sbe_jy = _mm512_load_ps(&sb_e_y[j]); - __m512 sb_jsq = _mm512_load_ps(&sb_sq[j]); + __m512 pjx = _mm512_loadu_ps(&pos_x[j]); + __m512 pjy = _mm512_loadu_ps(&pos_y[j]); + __m512 sbe_jx = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jy = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sb_jsq = _mm512_loadu_ps(&sb_sq[j]); __m512 r_ab_x = _mm512_sub_ps(pix, pjx); __m512 r_ab_y = _mm512_sub_ps(piy, pjy); diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2.cpp new file mode 100644 index 0000000..3c85829 --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2.cpp @@ -0,0 +1,519 @@ +// # Changes: Added ILP with k = 2 to j-loop, removed EPS checks where possible +// # Predecessor: variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 31) & ~31; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x_0 = _mm512_setzero_ps(); + __m512 acc_y_0 = _mm512_setzero_ps(); + __m512 acc_x_1 = _mm512_setzero_ps(); + __m512 acc_y_1 = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 32) { + __m512 j_idx_0 = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + __m512 j_idx_1 = _mm512_add_ps(_mm512_set1_ps((float)j + 16), lane_off); + + __mmask16 m_neq_i_0 = _mm512_cmp_ps_mask(j_idx_0, v_i, _CMP_NEQ_OQ); + __mmask16 m_neq_i_1 = _mm512_cmp_ps_mask(j_idx_1, v_i, _CMP_NEQ_OQ); + + __mmask16 m_in_range_0 = _mm512_cmp_ps_mask(j_idx_0, v_np, _CMP_LT_OQ); + __mmask16 m_in_range_1 = _mm512_cmp_ps_mask(j_idx_1, v_np, _CMP_LT_OQ); + + __mmask16 active_0 = m_neq_i_0 & m_in_range_0; + __mmask16 active_1 = m_neq_i_1 & m_in_range_1; + + __m512 pjx_0 = _mm512_loadu_ps(&pos_x[j]); + __m512 pjx_1 = _mm512_loadu_ps(&pos_x[j + 16]); + + + __m512 pjy_0 = _mm512_loadu_ps(&pos_y[j]); + __m512 pjy_1 = _mm512_loadu_ps(&pos_y[j + 16]); + + + __m512 sbe_jx_0 = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jx_1 = _mm512_loadu_ps(&sb_e_x[j + 16]); + + __m512 sbe_jy_0 = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sbe_jy_1 = _mm512_loadu_ps(&sb_e_y[j + 16]); + + __m512 sb_jsq_0 = _mm512_loadu_ps(&sb_sq[j]); + __m512 sb_jsq_1 = _mm512_loadu_ps(&sb_sq[j + 16]); + + __m512 r_ab_x_0 = _mm512_sub_ps(pix, pjx_0); + __m512 r_ab_x_1 = _mm512_sub_ps(pix, pjx_1); + + __m512 r_ab_y_0 = _mm512_sub_ps(piy, pjy_0); + __m512 r_ab_y_1 = _mm512_sub_ps(piy, pjy_1); + + __m512 r_ab_n2_0 = _mm512_fmadd_ps(r_ab_x_0, r_ab_x_0, _mm512_mul_ps(r_ab_y_0, r_ab_y_0)); + __m512 r_ab_n2_1 = _mm512_fmadd_ps(r_ab_x_1, r_ab_x_1, _mm512_mul_ps(r_ab_y_1, r_ab_y_1)); + + __m512 r_ab_n_0 = _mm512_sqrt_ps(r_ab_n2_0); + __m512 r_ab_n_1 = _mm512_sqrt_ps(r_ab_n2_1); + + __m512 r_ab2_x_0 = _mm512_sub_ps(r_ab_x_0, sbe_jx_0); + __m512 r_ab2_x_1 = _mm512_sub_ps(r_ab_x_1, sbe_jx_1); + + __m512 r_ab2_y_0 = _mm512_sub_ps(r_ab_y_0, sbe_jy_0); + __m512 r_ab2_y_1 = _mm512_sub_ps(r_ab_y_1, sbe_jy_1); + + __m512 r_ab2_n2_0 = _mm512_fmadd_ps(r_ab2_x_0, r_ab2_x_0, _mm512_mul_ps(r_ab2_y_0, r_ab2_y_0)); + __m512 r_ab2_n2_1 = _mm512_fmadd_ps(r_ab2_x_1, r_ab2_x_1, _mm512_mul_ps(r_ab2_y_1, r_ab2_y_1)); + + __m512 r_ab2_n_0 = _mm512_sqrt_ps(r_ab2_n2_0); + __m512 r_ab2_n_1 = _mm512_sqrt_ps(r_ab2_n2_1); + + __m512 r_sum_0 = _mm512_add_ps(r_ab_n_0, r_ab2_n_0); + __m512 r_sum_1 = _mm512_add_ps(r_ab_n_1, r_ab2_n_1); + + __m512 two_b_sq_0 = _mm512_fmsub_ps(r_sum_0, r_sum_0, sb_jsq_0); + __m512 two_b_sq_1 = _mm512_fmsub_ps(r_sum_1, r_sum_1, sb_jsq_1); + + __m512 two_b_0 = _mm512_sqrt_ps(two_b_sq_0); + __m512 two_b_1 = _mm512_sqrt_ps(two_b_sq_1); + + __m512 e_arg_0 = _mm512_mul_ps(two_b_0, v_neg_2s_inv); + __m512 e_arg_1 = _mm512_mul_ps(two_b_1, v_neg_2s_inv); + + e_arg_0 = _mm512_max_ps(e_arg_0, v_exp_min); + e_arg_1 = _mm512_max_ps(e_arg_1, v_exp_min); + + __m512 e_val_0 = _avx512_exp_ps(e_arg_0); + __m512 e_val_1 = _avx512_exp_ps(e_arg_1); + + __m512 numer_0 = _mm512_mul_ps(_mm512_mul_ps(r_sum_0, v_neg_v0_div_2s), e_val_0); + __m512 numer_1 = _mm512_mul_ps(_mm512_mul_ps(r_sum_1, v_neg_v0_div_2s), e_val_1); + + __m512 denom_0 = _mm512_mul_ps(_mm512_mul_ps(two_b_0, r_ab_n_0), r_ab2_n_0); + __m512 denom_1 = _mm512_mul_ps(_mm512_mul_ps(two_b_1, r_ab_n_1), r_ab2_n_1); + + __m512 denom_safe_0 = _mm512_mask_blend_ps(active_0, v_one, denom_0); + __m512 denom_safe_1 = _mm512_mask_blend_ps(active_1, v_one, denom_1); + + __m512 smc_0 = _mm512_div_ps(numer_0, denom_safe_0); + __m512 smc_1 = _mm512_div_ps(numer_1, denom_safe_1); + + __m512 sm_r_ab_0 = _mm512_mul_ps(smc_0, r_ab2_n_0); + __m512 sm_r_ab_1 = _mm512_mul_ps(smc_1, r_ab2_n_1); + + __m512 sm_r_ab2_0 = _mm512_mul_ps(smc_0, r_ab_n_0); + __m512 sm_r_ab2_1 = _mm512_mul_ps(smc_1, r_ab_n_1); + + __m512 f_x_0 = _mm512_fmadd_ps(r_ab2_x_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_x_0, sm_r_ab_0)); + __m512 f_x_1 = _mm512_fmadd_ps(r_ab2_x_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_x_1, sm_r_ab_1)); + + __m512 f_y_0 = _mm512_fmadd_ps(r_ab2_y_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_y_0, sm_r_ab_0)); + __m512 f_y_1 = _mm512_fmadd_ps(r_ab2_y_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_y_1, sm_r_ab_1)); + + __m512 f_norm_sq_0 = _mm512_fmadd_ps(f_x_0, f_x_0, _mm512_mul_ps(f_y_0, f_y_0)); + __m512 f_norm_sq_1 = _mm512_fmadd_ps(f_x_1, f_x_1, _mm512_mul_ps(f_y_1, f_y_1)); + + __m512 rhs_0 = _mm512_mul_ps(f_norm_sq_0, v_cos_phi_sq); + __m512 rhs_1 = _mm512_mul_ps(f_norm_sq_1, v_cos_phi_sq); + + __m512 e_dot_f_0 = _mm512_fmadd_ps(eix, f_x_0, _mm512_mul_ps(eiy, f_y_0)); + __m512 e_dot_f_1 = _mm512_fmadd_ps(eix, f_x_1, _mm512_mul_ps(eiy, f_y_1)); + + __m512 e_dot_f_sq_0 = _mm512_mul_ps(e_dot_f_0, e_dot_f_0); + __m512 e_dot_f_sq_1 = _mm512_mul_ps(e_dot_f_1, e_dot_f_1); + + __mmask16 c1_0 = _mm512_cmp_ps_mask(e_dot_f_0, v_zero, _CMP_LT_OQ); + __mmask16 c1_1 = _mm512_cmp_ps_mask(e_dot_f_1, v_zero, _CMP_LT_OQ); + + __mmask16 c2_0 = _mm512_cmp_ps_mask(e_dot_f_sq_0, rhs_0, _CMP_GT_OQ); + __mmask16 c2_1 = _mm512_cmp_ps_mask(e_dot_f_sq_1, rhs_1, _CMP_GT_OQ); + + __mmask16 cond_0 = c1_0 & c2_0; + __mmask16 cond_1 = c1_1 & c2_1; + + __m512 w_val_0 = _mm512_mask_blend_ps(cond_0, v_dt_neg, v_c_neg_dt); + __m512 w_val_1 = _mm512_mask_blend_ps(cond_1, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe_0 = _mm512_maskz_mov_ps(active_0, f_x_0); + __m512 f_x_safe_1 = _mm512_maskz_mov_ps(active_1, f_x_1); + + __m512 f_y_safe_0 = _mm512_maskz_mov_ps(active_0, f_y_0); + __m512 f_y_safe_1 = _mm512_maskz_mov_ps(active_1, f_y_1); + + acc_x_0 = _mm512_fmadd_ps(f_x_safe_0, w_val_0, acc_x_0); + acc_x_1 = _mm512_fmadd_ps(f_x_safe_1, w_val_1, acc_x_1); + + acc_y_0 = _mm512_fmadd_ps(f_y_safe_0, w_val_0, acc_y_0); + acc_y_1 = _mm512_fmadd_ps(f_y_safe_1, w_val_1, acc_y_1); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(_mm512_add_ps(acc_x_0, acc_x_1)); + f_alpha_dt_y += _mm512_reduce_add_ps(_mm512_add_ps(acc_y_0, acc_y_1)); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 16) { + + __m512 w_x_i_vec = _mm512_loadu_ps(&w_x[i]); + __m512 w_y_i_vec = _mm512_loadu_ps(&w_y[i]); + __m512 v_max_i_vec = _mm512_loadu_ps(&v_max[i]); + __m512 pos_x_i_vec = _mm512_loadu_ps(&pos_x[i]); + __m512 pos_y_i_vec = _mm512_loadu_ps(&pos_y[i]); + __m512 destination_y_i_vec = _mm512_loadu_ps(&destination_y[i]); + __m512 destination_x_i_vec = _mm512_loadu_ps(&destination_x[i]); + __m512 desired_v0_i_vec = _mm512_loadu_ps(&desired_v0[i]); + + __m512i color_i_vec = _mm512_loadu_si512((__m512i*)&color[i]); + + __m512 w_norm_sq = _mm512_fmadd_ps(w_x_i_vec, w_x_i_vec, _mm512_mul_ps(w_y_i_vec, w_y_i_vec)); + __m512 w_norm = _mm512_sqrt_ps(w_norm_sq); + __m512 correction_factor = _mm512_min_ps(_mm512_set1_ps(1.0f), _mm512_div_ps(v_max_i_vec, w_norm)); + + __m512 curr_vel_x_i_vec = _mm512_mul_ps(w_x_i_vec, correction_factor); + __m512 curr_vel_y_i_vec = _mm512_mul_ps(w_y_i_vec, correction_factor); + + __m512 dt_vec = _mm512_set1_ps(dt); + pos_x_i_vec = _mm512_fmadd_ps(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); + pos_y_i_vec = _mm512_fmadd_ps(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); + + __mmask16 cond_1_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_setzero_si512()); + __mmask16 cond_1_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.5f), _CMP_GT_OQ); + __mmask16 cond_1 = cond_1_left & cond_1_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_1, pos_x_i_vec, _mm512_set1_ps(1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_1, destination_y_i_vec, pos_y_i_vec); + w_x_i_vec = _mm512_mask_blend_ps(cond_1, w_x_i_vec, desired_v0_i_vec); + w_y_i_vec = _mm512_mask_blend_ps(cond_1, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_y_i_vec, w_y_i_vec); + + __mmask16 cond_2_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_set1_epi32(1)); + __mmask16 cond_2_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(1.5f), _CMP_LT_OQ); + __mmask16 cond_2 = cond_2_left & cond_2_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_2, pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_2, destination_y_i_vec, pos_y_i_vec); + __m512 neg_desired_v0 = _mm512_xor_ps(desired_v0_i_vec, _mm512_set1_ps(-0.0f)); + w_x_i_vec = _mm512_mask_blend_ps(cond_2, w_x_i_vec, neg_desired_v0); + w_y_i_vec = _mm512_mask_blend_ps(cond_2, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_y_i_vec, w_y_i_vec); + + pos_y_i_vec = _mm512_max_ps(pos_y_i_vec, _mm512_set1_ps(WALL_EPS)); + pos_y_i_vec = _mm512_min_ps(pos_y_i_vec, _mm512_set1_ps(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512 ex_vec = _mm512_sub_ps(destination_x_i_vec, pos_x_i_vec); + __m512 ey_vec = _mm512_sub_ps(destination_y_i_vec, pos_y_i_vec); + __m512 e_norm_sq = _mm512_fmadd_ps(ex_vec, ex_vec, _mm512_mul_ps(ey_vec, ey_vec)); + __m512 e_norm = _mm512_sqrt_ps(e_norm_sq); + __m512 e_norm_inv = _mm512_div_ps(_mm512_set1_ps(1.0f), e_norm); + __m512 e_x_i_vec = _mm512_mul_ps(ex_vec, e_norm_inv); + __m512 e_y_i_vec = _mm512_mul_ps(ey_vec, e_norm_inv); + + __m512 vel_norm_sq = _mm512_fmadd_ps(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_ps(curr_vel_y_i_vec, curr_vel_y_i_vec)); + __m512 sb_sq_i_vec = _mm512_mul_ps(vel_norm_sq, _mm512_set1_ps(delta_t_sq)); + __m512 vel_norm_dt = _mm512_sqrt_ps(sb_sq_i_vec); + + _mm512_storeu_ps(&curr_vel_x[i], curr_vel_x_i_vec); + _mm512_storeu_ps(&curr_vel_y[i], curr_vel_y_i_vec); + _mm512_storeu_ps(&w_x[i], w_x_i_vec); + _mm512_storeu_ps(&w_y[i], w_y_i_vec); + _mm512_storeu_ps(&pos_x[i], pos_x_i_vec); + _mm512_storeu_ps(&pos_y[i], pos_y_i_vec); + _mm512_storeu_ps(&e_x[i], e_x_i_vec); + _mm512_storeu_ps(&e_y[i], e_y_i_vec); + _mm512_storeu_ps(&destination_y[i], destination_y_i_vec); + _mm512_storeu_ps(&sb_sq[i], sb_sq_i_vec); + _mm512_storeu_ps(&sb_e_x[i], _mm512_mul_ps(e_x_i_vec, vel_norm_dt)); + _mm512_storeu_ps(&sb_e_y[i], _mm512_mul_ps(e_y_i_vec, vel_norm_dt)); + } + + +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_moreilp.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_moreilp.cpp new file mode 100644 index 0000000..4a2aa0e --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_moreilp.cpp @@ -0,0 +1,619 @@ +// # Changes: Added ILP with k = 2 to second pass +// # Predecessor: variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 31) & ~31; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x_0 = _mm512_setzero_ps(); + __m512 acc_y_0 = _mm512_setzero_ps(); + __m512 acc_x_1 = _mm512_setzero_ps(); + __m512 acc_y_1 = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 32) { + __m512 j_idx_0 = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + __m512 j_idx_1 = _mm512_add_ps(_mm512_set1_ps((float)j + 16), lane_off); + + __mmask16 m_neq_i_0 = _mm512_cmp_ps_mask(j_idx_0, v_i, _CMP_NEQ_OQ); + __mmask16 m_neq_i_1 = _mm512_cmp_ps_mask(j_idx_1, v_i, _CMP_NEQ_OQ); + + __mmask16 m_in_range_0 = _mm512_cmp_ps_mask(j_idx_0, v_np, _CMP_LT_OQ); + __mmask16 m_in_range_1 = _mm512_cmp_ps_mask(j_idx_1, v_np, _CMP_LT_OQ); + + __mmask16 active_0 = m_neq_i_0 & m_in_range_0; + __mmask16 active_1 = m_neq_i_1 & m_in_range_1; + + __m512 pjx_0 = _mm512_loadu_ps(&pos_x[j]); + __m512 pjx_1 = _mm512_loadu_ps(&pos_x[j + 16]); + + __m512 pjy_0 = _mm512_loadu_ps(&pos_y[j]); + __m512 pjy_1 = _mm512_loadu_ps(&pos_y[j + 16]); + + __m512 sbe_jx_0 = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jx_1 = _mm512_loadu_ps(&sb_e_x[j + 16]); + + __m512 sbe_jy_0 = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sbe_jy_1 = _mm512_loadu_ps(&sb_e_y[j + 16]); + + __m512 sb_jsq_0 = _mm512_loadu_ps(&sb_sq[j]); + __m512 sb_jsq_1 = _mm512_loadu_ps(&sb_sq[j + 16]); + + __m512 r_ab_x_0 = _mm512_sub_ps(pix, pjx_0); + __m512 r_ab_x_1 = _mm512_sub_ps(pix, pjx_1); + + __m512 r_ab_y_0 = _mm512_sub_ps(piy, pjy_0); + __m512 r_ab_y_1 = _mm512_sub_ps(piy, pjy_1); + + __m512 r_ab_n2_0 = _mm512_fmadd_ps(r_ab_x_0, r_ab_x_0, _mm512_mul_ps(r_ab_y_0, r_ab_y_0)); + __m512 r_ab_n2_1 = _mm512_fmadd_ps(r_ab_x_1, r_ab_x_1, _mm512_mul_ps(r_ab_y_1, r_ab_y_1)); + + __m512 r_ab_n_0 = _mm512_sqrt_ps(r_ab_n2_0); + __m512 r_ab_n_1 = _mm512_sqrt_ps(r_ab_n2_1); + + __m512 r_ab2_x_0 = _mm512_sub_ps(r_ab_x_0, sbe_jx_0); + __m512 r_ab2_x_1 = _mm512_sub_ps(r_ab_x_1, sbe_jx_1); + + __m512 r_ab2_y_0 = _mm512_sub_ps(r_ab_y_0, sbe_jy_0); + __m512 r_ab2_y_1 = _mm512_sub_ps(r_ab_y_1, sbe_jy_1); + + __m512 r_ab2_n2_0 = _mm512_fmadd_ps(r_ab2_x_0, r_ab2_x_0, _mm512_mul_ps(r_ab2_y_0, r_ab2_y_0)); + __m512 r_ab2_n2_1 = _mm512_fmadd_ps(r_ab2_x_1, r_ab2_x_1, _mm512_mul_ps(r_ab2_y_1, r_ab2_y_1)); + + __m512 r_ab2_n_0 = _mm512_sqrt_ps(r_ab2_n2_0); + __m512 r_ab2_n_1 = _mm512_sqrt_ps(r_ab2_n2_1); + + __m512 r_sum_0 = _mm512_add_ps(r_ab_n_0, r_ab2_n_0); + __m512 r_sum_1 = _mm512_add_ps(r_ab_n_1, r_ab2_n_1); + + __m512 two_b_sq_0 = _mm512_fmsub_ps(r_sum_0, r_sum_0, sb_jsq_0); + __m512 two_b_sq_1 = _mm512_fmsub_ps(r_sum_1, r_sum_1, sb_jsq_1); + + __m512 two_b_0 = _mm512_sqrt_ps(two_b_sq_0); + __m512 two_b_1 = _mm512_sqrt_ps(two_b_sq_1); + + __m512 e_arg_0 = _mm512_mul_ps(two_b_0, v_neg_2s_inv); + __m512 e_arg_1 = _mm512_mul_ps(two_b_1, v_neg_2s_inv); + + e_arg_0 = _mm512_max_ps(e_arg_0, v_exp_min); + e_arg_1 = _mm512_max_ps(e_arg_1, v_exp_min); + + __m512 e_val_0 = _avx512_exp_ps(e_arg_0); + __m512 e_val_1 = _avx512_exp_ps(e_arg_1); + + __m512 numer_0 = _mm512_mul_ps(_mm512_mul_ps(r_sum_0, v_neg_v0_div_2s), e_val_0); + __m512 numer_1 = _mm512_mul_ps(_mm512_mul_ps(r_sum_1, v_neg_v0_div_2s), e_val_1); + + __m512 denom_0 = _mm512_mul_ps(_mm512_mul_ps(two_b_0, r_ab_n_0), r_ab2_n_0); + __m512 denom_1 = _mm512_mul_ps(_mm512_mul_ps(two_b_1, r_ab_n_1), r_ab2_n_1); + + __m512 denom_safe_0 = _mm512_mask_blend_ps(active_0, v_one, denom_0); + __m512 denom_safe_1 = _mm512_mask_blend_ps(active_1, v_one, denom_1); + + __m512 smc_0 = _mm512_div_ps(numer_0, denom_safe_0); + __m512 smc_1 = _mm512_div_ps(numer_1, denom_safe_1); + + __m512 sm_r_ab_0 = _mm512_mul_ps(smc_0, r_ab2_n_0); + __m512 sm_r_ab_1 = _mm512_mul_ps(smc_1, r_ab2_n_1); + + __m512 sm_r_ab2_0 = _mm512_mul_ps(smc_0, r_ab_n_0); + __m512 sm_r_ab2_1 = _mm512_mul_ps(smc_1, r_ab_n_1); + + __m512 f_x_0 = _mm512_fmadd_ps(r_ab2_x_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_x_0, sm_r_ab_0)); + __m512 f_x_1 = _mm512_fmadd_ps(r_ab2_x_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_x_1, sm_r_ab_1)); + + __m512 f_y_0 = _mm512_fmadd_ps(r_ab2_y_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_y_0, sm_r_ab_0)); + __m512 f_y_1 = _mm512_fmadd_ps(r_ab2_y_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_y_1, sm_r_ab_1)); + + __m512 f_norm_sq_0 = _mm512_fmadd_ps(f_x_0, f_x_0, _mm512_mul_ps(f_y_0, f_y_0)); + __m512 f_norm_sq_1 = _mm512_fmadd_ps(f_x_1, f_x_1, _mm512_mul_ps(f_y_1, f_y_1)); + + __m512 rhs_0 = _mm512_mul_ps(f_norm_sq_0, v_cos_phi_sq); + __m512 rhs_1 = _mm512_mul_ps(f_norm_sq_1, v_cos_phi_sq); + + __m512 e_dot_f_0 = _mm512_fmadd_ps(eix, f_x_0, _mm512_mul_ps(eiy, f_y_0)); + __m512 e_dot_f_1 = _mm512_fmadd_ps(eix, f_x_1, _mm512_mul_ps(eiy, f_y_1)); + + __m512 e_dot_f_sq_0 = _mm512_mul_ps(e_dot_f_0, e_dot_f_0); + __m512 e_dot_f_sq_1 = _mm512_mul_ps(e_dot_f_1, e_dot_f_1); + + __mmask16 c1_0 = _mm512_cmp_ps_mask(e_dot_f_0, v_zero, _CMP_LT_OQ); + __mmask16 c1_1 = _mm512_cmp_ps_mask(e_dot_f_1, v_zero, _CMP_LT_OQ); + + __mmask16 c2_0 = _mm512_cmp_ps_mask(e_dot_f_sq_0, rhs_0, _CMP_GT_OQ); + __mmask16 c2_1 = _mm512_cmp_ps_mask(e_dot_f_sq_1, rhs_1, _CMP_GT_OQ); + + __mmask16 cond_0 = c1_0 & c2_0; + __mmask16 cond_1 = c1_1 & c2_1; + + __m512 w_val_0 = _mm512_mask_blend_ps(cond_0, v_dt_neg, v_c_neg_dt); + __m512 w_val_1 = _mm512_mask_blend_ps(cond_1, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe_0 = _mm512_maskz_mov_ps(active_0, f_x_0); + __m512 f_x_safe_1 = _mm512_maskz_mov_ps(active_1, f_x_1); + + __m512 f_y_safe_0 = _mm512_maskz_mov_ps(active_0, f_y_0); + __m512 f_y_safe_1 = _mm512_maskz_mov_ps(active_1, f_y_1); + + acc_x_0 = _mm512_fmadd_ps(f_x_safe_0, w_val_0, acc_x_0); + acc_x_1 = _mm512_fmadd_ps(f_x_safe_1, w_val_1, acc_x_1); + + acc_y_0 = _mm512_fmadd_ps(f_y_safe_0, w_val_0, acc_y_0); + acc_y_1 = _mm512_fmadd_ps(f_y_safe_1, w_val_1, acc_y_1); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(_mm512_add_ps(acc_x_0, acc_x_1)); + f_alpha_dt_y += _mm512_reduce_add_ps(_mm512_add_ps(acc_y_0, acc_y_1)); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 32) { + + __m512 w_x_i_vec_0 = _mm512_loadu_ps(&w_x[i]); + __m512 w_x_i_vec_1 = _mm512_loadu_ps(&w_x[i + 16]); + + __m512 w_y_i_vec_0 = _mm512_loadu_ps(&w_y[i]); + __m512 w_y_i_vec_1 = _mm512_loadu_ps(&w_y[i + 16]); + + __m512 v_max_i_vec_0 = _mm512_loadu_ps(&v_max[i]); + __m512 v_max_i_vec_1 = _mm512_loadu_ps(&v_max[i + 16]); + + __m512 pos_x_i_vec_0 = _mm512_loadu_ps(&pos_x[i]); + __m512 pos_x_i_vec_1 = _mm512_loadu_ps(&pos_x[i + 16]); + + __m512 pos_y_i_vec_0 = _mm512_loadu_ps(&pos_y[i]); + __m512 pos_y_i_vec_1 = _mm512_loadu_ps(&pos_y[i + 16]); + + __m512 destination_y_i_vec_0 = _mm512_loadu_ps(&destination_y[i]); + __m512 destination_y_i_vec_1 = _mm512_loadu_ps(&destination_y[i + 16]); + + __m512 destination_x_i_vec_0 = _mm512_loadu_ps(&destination_x[i]); + __m512 destination_x_i_vec_1 = _mm512_loadu_ps(&destination_x[i + 16]); + + __m512 desired_v0_i_vec_0 = _mm512_loadu_ps(&desired_v0[i]); + __m512 desired_v0_i_vec_1 = _mm512_loadu_ps(&desired_v0[i + 16]); + + __m512i color_i_vec_0 = _mm512_loadu_si512((__m512i *)&color[i]); + __m512i color_i_vec_1 = _mm512_loadu_si512((__m512i *)&color[i + 16]); + + __m512 w_norm_sq_0 = _mm512_fmadd_ps(w_x_i_vec_0, w_x_i_vec_0, _mm512_mul_ps(w_y_i_vec_0, w_y_i_vec_0)); + __m512 w_norm_sq_1 = _mm512_fmadd_ps(w_x_i_vec_1, w_x_i_vec_1, _mm512_mul_ps(w_y_i_vec_1, w_y_i_vec_1)); + + __m512 w_norm_0 = _mm512_sqrt_ps(w_norm_sq_0); + __m512 w_norm_1 = _mm512_sqrt_ps(w_norm_sq_1); + + __m512 correction_factor_0 = _mm512_min_ps(_mm512_set1_ps(1.0f), _mm512_div_ps(v_max_i_vec_0, w_norm_0)); + __m512 correction_factor_1 = _mm512_min_ps(_mm512_set1_ps(1.0f), _mm512_div_ps(v_max_i_vec_1, w_norm_1)); + + __m512 curr_vel_x_i_vec_0 = _mm512_mul_ps(w_x_i_vec_0, correction_factor_0); + __m512 curr_vel_x_i_vec_1 = _mm512_mul_ps(w_x_i_vec_1, correction_factor_1); + + __m512 curr_vel_y_i_vec_0 = _mm512_mul_ps(w_y_i_vec_0, correction_factor_0); + __m512 curr_vel_y_i_vec_1 = _mm512_mul_ps(w_y_i_vec_1, correction_factor_1); + + __m512 dt_vec = _mm512_set1_ps(dt); + + pos_x_i_vec_0 = _mm512_fmadd_ps(curr_vel_x_i_vec_0, dt_vec, pos_x_i_vec_0); + pos_x_i_vec_1 = _mm512_fmadd_ps(curr_vel_x_i_vec_1, dt_vec, pos_x_i_vec_1); + + pos_y_i_vec_0 = _mm512_fmadd_ps(curr_vel_y_i_vec_0, dt_vec, pos_y_i_vec_0); + pos_y_i_vec_1 = _mm512_fmadd_ps(curr_vel_y_i_vec_1, dt_vec, pos_y_i_vec_1); + + __mmask16 cond_1_left_0 = _mm512_cmpeq_epi32_mask(color_i_vec_0, _mm512_setzero_si512()); + __mmask16 cond_1_left_1 = _mm512_cmpeq_epi32_mask(color_i_vec_1, _mm512_setzero_si512()); + + __mmask16 cond_1_right_0 = _mm512_cmp_ps_mask(pos_x_i_vec_0, _mm512_set1_ps(BOARD_LENGTH - 1.5f), _CMP_GT_OQ); + __mmask16 cond_1_right_1 = _mm512_cmp_ps_mask(pos_x_i_vec_1, _mm512_set1_ps(BOARD_LENGTH - 1.5f), _CMP_GT_OQ); + + __mmask16 cond_1_0 = cond_1_left_0 & cond_1_right_0; + __mmask16 cond_1_1 = cond_1_left_1 & cond_1_right_1; + + pos_x_i_vec_0 = _mm512_mask_blend_ps(cond_1_0, pos_x_i_vec_0, _mm512_set1_ps(1.0f)); + pos_x_i_vec_1 = _mm512_mask_blend_ps(cond_1_1, pos_x_i_vec_1, _mm512_set1_ps(1.0f)); + + destination_y_i_vec_0 = _mm512_mask_blend_ps(cond_1_0, destination_y_i_vec_0, pos_y_i_vec_0); + destination_y_i_vec_1 = _mm512_mask_blend_ps(cond_1_1, destination_y_i_vec_1, pos_y_i_vec_1); + + w_x_i_vec_0 = _mm512_mask_blend_ps(cond_1_0, w_x_i_vec_0, desired_v0_i_vec_0); + w_x_i_vec_1 = _mm512_mask_blend_ps(cond_1_1, w_x_i_vec_1, desired_v0_i_vec_1); + + w_y_i_vec_0 = _mm512_mask_blend_ps(cond_1_0, w_y_i_vec_0, _mm512_setzero_ps()); + w_y_i_vec_1 = _mm512_mask_blend_ps(cond_1_1, w_y_i_vec_1, _mm512_setzero_ps()); + + curr_vel_x_i_vec_0 = _mm512_mask_blend_ps(cond_1_0, curr_vel_x_i_vec_0, w_x_i_vec_0); + curr_vel_x_i_vec_1 = _mm512_mask_blend_ps(cond_1_1, curr_vel_x_i_vec_1, w_x_i_vec_1); + + curr_vel_y_i_vec_0 = _mm512_mask_blend_ps(cond_1_0, curr_vel_y_i_vec_0, w_y_i_vec_0); + curr_vel_y_i_vec_1 = _mm512_mask_blend_ps(cond_1_1, curr_vel_y_i_vec_1, w_y_i_vec_1); + + __mmask16 cond_2_left_0 = _mm512_cmpeq_epi32_mask(color_i_vec_0, _mm512_set1_epi32(1)); + __mmask16 cond_2_left_1 = _mm512_cmpeq_epi32_mask(color_i_vec_1, _mm512_set1_epi32(1)); + + __mmask16 cond_2_right_0 = _mm512_cmp_ps_mask(pos_x_i_vec_0, _mm512_set1_ps(1.5f), _CMP_LT_OQ); + __mmask16 cond_2_right_1 = _mm512_cmp_ps_mask(pos_x_i_vec_1, _mm512_set1_ps(1.5f), _CMP_LT_OQ); + + __mmask16 cond_2_0 = cond_2_left_0 & cond_2_right_0; + __mmask16 cond_2_1 = cond_2_left_1 & cond_2_right_1; + + pos_x_i_vec_0 = _mm512_mask_blend_ps(cond_2_0, pos_x_i_vec_0, _mm512_set1_ps(BOARD_LENGTH - 1.0f)); + pos_x_i_vec_1 = _mm512_mask_blend_ps(cond_2_1, pos_x_i_vec_1, _mm512_set1_ps(BOARD_LENGTH - 1.0f)); + + destination_y_i_vec_0 = _mm512_mask_blend_ps(cond_2_0, destination_y_i_vec_0, pos_y_i_vec_0); + destination_y_i_vec_1 = _mm512_mask_blend_ps(cond_2_1, destination_y_i_vec_1, pos_y_i_vec_1); + + __m512 neg_desired_v0_0 = _mm512_xor_ps(desired_v0_i_vec_0, _mm512_set1_ps(-0.0f)); + __m512 neg_desired_v0_1 = _mm512_xor_ps(desired_v0_i_vec_1, _mm512_set1_ps(-0.0f)); + + w_x_i_vec_0 = _mm512_mask_blend_ps(cond_2_0, w_x_i_vec_0, neg_desired_v0_0); + w_x_i_vec_1 = _mm512_mask_blend_ps(cond_2_1, w_x_i_vec_1, neg_desired_v0_1); + + w_y_i_vec_0 = _mm512_mask_blend_ps(cond_2_0, w_y_i_vec_0, _mm512_setzero_ps()); + w_y_i_vec_1 = _mm512_mask_blend_ps(cond_2_1, w_y_i_vec_1, _mm512_setzero_ps()); + + curr_vel_x_i_vec_0 = _mm512_mask_blend_ps(cond_2_0, curr_vel_x_i_vec_0, w_x_i_vec_0); + curr_vel_y_i_vec_1 = _mm512_mask_blend_ps(cond_2_1, curr_vel_y_i_vec_1, w_y_i_vec_1); + + pos_y_i_vec_0 = _mm512_max_ps(pos_y_i_vec_0, _mm512_set1_ps(WALL_EPS)); + pos_y_i_vec_1 = _mm512_max_ps(pos_y_i_vec_1, _mm512_set1_ps(WALL_EPS)); + + pos_y_i_vec_0 = _mm512_min_ps(pos_y_i_vec_0, _mm512_set1_ps(BOARD_WIDTH_MINUS_WALL_EPS)); + pos_y_i_vec_1 = _mm512_min_ps(pos_y_i_vec_1, _mm512_set1_ps(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512 ex_vec_0 = _mm512_sub_ps(destination_x_i_vec_0, pos_x_i_vec_0); + __m512 ex_vec_1 = _mm512_sub_ps(destination_x_i_vec_1, pos_x_i_vec_1); + + __m512 ey_vec_0 = _mm512_sub_ps(destination_y_i_vec_0, pos_y_i_vec_0); + __m512 ey_vec_1 = _mm512_sub_ps(destination_y_i_vec_1, pos_y_i_vec_1); + + __m512 e_norm_sq_0 = _mm512_fmadd_ps(ex_vec_0, ex_vec_0, _mm512_mul_ps(ey_vec_0, ey_vec_0)); + __m512 e_norm_sq_1 = _mm512_fmadd_ps(ex_vec_1, ex_vec_1, _mm512_mul_ps(ey_vec_1, ey_vec_1)); + + __m512 e_norm_0 = _mm512_sqrt_ps(e_norm_sq_0); + __m512 e_norm_1 = _mm512_sqrt_ps(e_norm_sq_1); + + __m512 e_norm_inv_0 = _mm512_div_ps(_mm512_set1_ps(1.0f), e_norm_0); + __m512 e_norm_inv_1 = _mm512_div_ps(_mm512_set1_ps(1.0f), e_norm_1); + + __m512 e_x_i_vec_0 = _mm512_mul_ps(ex_vec_0, e_norm_inv_0); + __m512 e_x_i_vec_1 = _mm512_mul_ps(ex_vec_1, e_norm_inv_1); + + __m512 e_y_i_vec_0 = _mm512_mul_ps(ey_vec_0, e_norm_inv_0); + __m512 e_y_i_vec_1 = _mm512_mul_ps(ey_vec_1, e_norm_inv_1); + + __m512 vel_norm_sq_0 = _mm512_fmadd_ps(curr_vel_x_i_vec_0, curr_vel_x_i_vec_0, _mm512_mul_ps(curr_vel_y_i_vec_0, curr_vel_y_i_vec_0)); + __m512 vel_norm_sq_1 = _mm512_fmadd_ps(curr_vel_x_i_vec_1, curr_vel_x_i_vec_1, _mm512_mul_ps(curr_vel_y_i_vec_1, curr_vel_y_i_vec_1)); + + __m512 sb_sq_i_vec_0 = _mm512_mul_ps(vel_norm_sq_0, _mm512_set1_ps(delta_t_sq)); + __m512 sb_sq_i_vec_1 = _mm512_mul_ps(vel_norm_sq_1, _mm512_set1_ps(delta_t_sq)); + + __m512 vel_norm_dt_0 = _mm512_sqrt_ps(sb_sq_i_vec_0); + __m512 vel_norm_dt_1 = _mm512_sqrt_ps(sb_sq_i_vec_1); + + _mm512_storeu_ps(&curr_vel_x[i], curr_vel_x_i_vec_0); + _mm512_storeu_ps(&curr_vel_x[i + 16], curr_vel_x_i_vec_1); + + _mm512_storeu_ps(&curr_vel_y[i], curr_vel_y_i_vec_0); + _mm512_storeu_ps(&curr_vel_y[i + 16], curr_vel_y_i_vec_1); + + _mm512_storeu_ps(&w_x[i], w_x_i_vec_0); + _mm512_storeu_ps(&w_x[i + 16], w_x_i_vec_1); + + _mm512_storeu_ps(&w_y[i], w_y_i_vec_0); + _mm512_storeu_ps(&w_y[i + 16], w_y_i_vec_1); + + _mm512_storeu_ps(&pos_x[i], pos_x_i_vec_0); + _mm512_storeu_ps(&pos_x[i + 16], pos_x_i_vec_1); + + _mm512_storeu_ps(&pos_y[i], pos_y_i_vec_0); + _mm512_storeu_ps(&pos_y[i + 16], pos_y_i_vec_1); + + _mm512_storeu_ps(&e_x[i], e_x_i_vec_0); + _mm512_storeu_ps(&e_x[i + 16], e_x_i_vec_1); + + _mm512_storeu_ps(&e_y[i], e_y_i_vec_0); + _mm512_storeu_ps(&e_y[i + 16], e_y_i_vec_1); + + _mm512_storeu_ps(&destination_y[i], destination_y_i_vec_0); + _mm512_storeu_ps(&destination_y[i + 16], destination_y_i_vec_1); + + _mm512_storeu_ps(&sb_sq[i], sb_sq_i_vec_0); + _mm512_storeu_ps(&sb_sq[i + 16], sb_sq_i_vec_1); + + _mm512_storeu_ps(&sb_e_x[i], _mm512_mul_ps(e_x_i_vec_0, vel_norm_dt_0)); + _mm512_storeu_ps(&sb_e_x[i + 16], _mm512_mul_ps(e_x_i_vec_1, vel_norm_dt_1)); + + _mm512_storeu_ps(&sb_e_y[i], _mm512_mul_ps(e_y_i_vec_0, vel_norm_dt_0)); + _mm512_storeu_ps(&sb_e_y[i + 16], _mm512_mul_ps(e_y_i_vec_1, vel_norm_dt_1)); + } + +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_rsqrt.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_rsqrt.cpp new file mode 100644 index 0000000..5178445 --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2_rsqrt.cpp @@ -0,0 +1,515 @@ +// # Changes: Used _mm512_rsqrt14_ps() instead of _mm512_sqrt_ps() and _mm512_div_ps() +// # Predecessor: variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 31) & ~31; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x_0 = _mm512_setzero_ps(); + __m512 acc_y_0 = _mm512_setzero_ps(); + __m512 acc_x_1 = _mm512_setzero_ps(); + __m512 acc_y_1 = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 32) { + __m512 j_idx_0 = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + __m512 j_idx_1 = _mm512_add_ps(_mm512_set1_ps((float)j + 16), lane_off); + + __mmask16 m_neq_i_0 = _mm512_cmp_ps_mask(j_idx_0, v_i, _CMP_NEQ_OQ); + __mmask16 m_neq_i_1 = _mm512_cmp_ps_mask(j_idx_1, v_i, _CMP_NEQ_OQ); + + __mmask16 m_in_range_0 = _mm512_cmp_ps_mask(j_idx_0, v_np, _CMP_LT_OQ); + __mmask16 m_in_range_1 = _mm512_cmp_ps_mask(j_idx_1, v_np, _CMP_LT_OQ); + + __mmask16 active_0 = m_neq_i_0 & m_in_range_0; + __mmask16 active_1 = m_neq_i_1 & m_in_range_1; + + __m512 pjx_0 = _mm512_loadu_ps(&pos_x[j]); + __m512 pjx_1 = _mm512_loadu_ps(&pos_x[j + 16]); + + __m512 pjy_0 = _mm512_loadu_ps(&pos_y[j]); + __m512 pjy_1 = _mm512_loadu_ps(&pos_y[j + 16]); + + __m512 sbe_jx_0 = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jx_1 = _mm512_loadu_ps(&sb_e_x[j + 16]); + + __m512 sbe_jy_0 = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sbe_jy_1 = _mm512_loadu_ps(&sb_e_y[j + 16]); + + __m512 sb_jsq_0 = _mm512_loadu_ps(&sb_sq[j]); + __m512 sb_jsq_1 = _mm512_loadu_ps(&sb_sq[j + 16]); + + __m512 r_ab_x_0 = _mm512_sub_ps(pix, pjx_0); + __m512 r_ab_x_1 = _mm512_sub_ps(pix, pjx_1); + + __m512 r_ab_y_0 = _mm512_sub_ps(piy, pjy_0); + __m512 r_ab_y_1 = _mm512_sub_ps(piy, pjy_1); + + __m512 r_ab_n2_0 = _mm512_fmadd_ps(r_ab_x_0, r_ab_x_0, _mm512_mul_ps(r_ab_y_0, r_ab_y_0)); + __m512 r_ab_n2_1 = _mm512_fmadd_ps(r_ab_x_1, r_ab_x_1, _mm512_mul_ps(r_ab_y_1, r_ab_y_1)); + + __m512 r_ab_n_0 = _mm512_sqrt_ps(r_ab_n2_0); + __m512 r_ab_n_1 = _mm512_sqrt_ps(r_ab_n2_1); + + __m512 r_ab2_x_0 = _mm512_sub_ps(r_ab_x_0, sbe_jx_0); + __m512 r_ab2_x_1 = _mm512_sub_ps(r_ab_x_1, sbe_jx_1); + + __m512 r_ab2_y_0 = _mm512_sub_ps(r_ab_y_0, sbe_jy_0); + __m512 r_ab2_y_1 = _mm512_sub_ps(r_ab_y_1, sbe_jy_1); + + __m512 r_ab2_n2_0 = _mm512_fmadd_ps(r_ab2_x_0, r_ab2_x_0, _mm512_mul_ps(r_ab2_y_0, r_ab2_y_0)); + __m512 r_ab2_n2_1 = _mm512_fmadd_ps(r_ab2_x_1, r_ab2_x_1, _mm512_mul_ps(r_ab2_y_1, r_ab2_y_1)); + + __m512 r_ab2_n_0 = _mm512_sqrt_ps(r_ab2_n2_0); + __m512 r_ab2_n_1 = _mm512_sqrt_ps(r_ab2_n2_1); + + __m512 r_sum_0 = _mm512_add_ps(r_ab_n_0, r_ab2_n_0); + __m512 r_sum_1 = _mm512_add_ps(r_ab_n_1, r_ab2_n_1); + + __m512 two_b_sq_0 = _mm512_fmsub_ps(r_sum_0, r_sum_0, sb_jsq_0); + __m512 two_b_sq_1 = _mm512_fmsub_ps(r_sum_1, r_sum_1, sb_jsq_1); + + __m512 two_b_0 = _mm512_sqrt_ps(two_b_sq_0); + __m512 two_b_1 = _mm512_sqrt_ps(two_b_sq_1); + + __m512 e_arg_0 = _mm512_mul_ps(two_b_0, v_neg_2s_inv); + __m512 e_arg_1 = _mm512_mul_ps(two_b_1, v_neg_2s_inv); + + e_arg_0 = _mm512_max_ps(e_arg_0, v_exp_min); + e_arg_1 = _mm512_max_ps(e_arg_1, v_exp_min); + + __m512 e_val_0 = _avx512_exp_ps(e_arg_0); + __m512 e_val_1 = _avx512_exp_ps(e_arg_1); + + __m512 numer_0 = _mm512_mul_ps(_mm512_mul_ps(r_sum_0, v_neg_v0_div_2s), e_val_0); + __m512 numer_1 = _mm512_mul_ps(_mm512_mul_ps(r_sum_1, v_neg_v0_div_2s), e_val_1); + + __m512 denom_0 = _mm512_mul_ps(_mm512_mul_ps(two_b_0, r_ab_n_0), r_ab2_n_0); + __m512 denom_1 = _mm512_mul_ps(_mm512_mul_ps(two_b_1, r_ab_n_1), r_ab2_n_1); + + __m512 denom_safe_0 = _mm512_mask_blend_ps(active_0, v_one, denom_0); + __m512 denom_safe_1 = _mm512_mask_blend_ps(active_1, v_one, denom_1); + + __m512 smc_0 = _mm512_div_ps(numer_0, denom_safe_0); + __m512 smc_1 = _mm512_div_ps(numer_1, denom_safe_1); + + __m512 sm_r_ab_0 = _mm512_mul_ps(smc_0, r_ab2_n_0); + __m512 sm_r_ab_1 = _mm512_mul_ps(smc_1, r_ab2_n_1); + + __m512 sm_r_ab2_0 = _mm512_mul_ps(smc_0, r_ab_n_0); + __m512 sm_r_ab2_1 = _mm512_mul_ps(smc_1, r_ab_n_1); + + __m512 f_x_0 = _mm512_fmadd_ps(r_ab2_x_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_x_0, sm_r_ab_0)); + __m512 f_x_1 = _mm512_fmadd_ps(r_ab2_x_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_x_1, sm_r_ab_1)); + + __m512 f_y_0 = _mm512_fmadd_ps(r_ab2_y_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_y_0, sm_r_ab_0)); + __m512 f_y_1 = _mm512_fmadd_ps(r_ab2_y_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_y_1, sm_r_ab_1)); + + __m512 f_norm_sq_0 = _mm512_fmadd_ps(f_x_0, f_x_0, _mm512_mul_ps(f_y_0, f_y_0)); + __m512 f_norm_sq_1 = _mm512_fmadd_ps(f_x_1, f_x_1, _mm512_mul_ps(f_y_1, f_y_1)); + + __m512 rhs_0 = _mm512_mul_ps(f_norm_sq_0, v_cos_phi_sq); + __m512 rhs_1 = _mm512_mul_ps(f_norm_sq_1, v_cos_phi_sq); + + __m512 e_dot_f_0 = _mm512_fmadd_ps(eix, f_x_0, _mm512_mul_ps(eiy, f_y_0)); + __m512 e_dot_f_1 = _mm512_fmadd_ps(eix, f_x_1, _mm512_mul_ps(eiy, f_y_1)); + + __m512 e_dot_f_sq_0 = _mm512_mul_ps(e_dot_f_0, e_dot_f_0); + __m512 e_dot_f_sq_1 = _mm512_mul_ps(e_dot_f_1, e_dot_f_1); + + __mmask16 c1_0 = _mm512_cmp_ps_mask(e_dot_f_0, v_zero, _CMP_LT_OQ); + __mmask16 c1_1 = _mm512_cmp_ps_mask(e_dot_f_1, v_zero, _CMP_LT_OQ); + + __mmask16 c2_0 = _mm512_cmp_ps_mask(e_dot_f_sq_0, rhs_0, _CMP_GT_OQ); + __mmask16 c2_1 = _mm512_cmp_ps_mask(e_dot_f_sq_1, rhs_1, _CMP_GT_OQ); + + __mmask16 cond_0 = c1_0 & c2_0; + __mmask16 cond_1 = c1_1 & c2_1; + + __m512 w_val_0 = _mm512_mask_blend_ps(cond_0, v_dt_neg, v_c_neg_dt); + __m512 w_val_1 = _mm512_mask_blend_ps(cond_1, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe_0 = _mm512_maskz_mov_ps(active_0, f_x_0); + __m512 f_x_safe_1 = _mm512_maskz_mov_ps(active_1, f_x_1); + + __m512 f_y_safe_0 = _mm512_maskz_mov_ps(active_0, f_y_0); + __m512 f_y_safe_1 = _mm512_maskz_mov_ps(active_1, f_y_1); + + acc_x_0 = _mm512_fmadd_ps(f_x_safe_0, w_val_0, acc_x_0); + acc_x_1 = _mm512_fmadd_ps(f_x_safe_1, w_val_1, acc_x_1); + + acc_y_0 = _mm512_fmadd_ps(f_y_safe_0, w_val_0, acc_y_0); + acc_y_1 = _mm512_fmadd_ps(f_y_safe_1, w_val_1, acc_y_1); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(_mm512_add_ps(acc_x_0, acc_x_1)); + f_alpha_dt_y += _mm512_reduce_add_ps(_mm512_add_ps(acc_y_0, acc_y_1)); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 16) { + + __m512 w_x_i_vec = _mm512_loadu_ps(&w_x[i]); + __m512 w_y_i_vec = _mm512_loadu_ps(&w_y[i]); + __m512 v_max_i_vec = _mm512_loadu_ps(&v_max[i]); + __m512 pos_x_i_vec = _mm512_loadu_ps(&pos_x[i]); + __m512 pos_y_i_vec = _mm512_loadu_ps(&pos_y[i]); + __m512 destination_y_i_vec = _mm512_loadu_ps(&destination_y[i]); + __m512 destination_x_i_vec = _mm512_loadu_ps(&destination_x[i]); + __m512 desired_v0_i_vec = _mm512_loadu_ps(&desired_v0[i]); + + __m512i color_i_vec = _mm512_loadu_si512((__m512i *)&color[i]); + + __m512 w_norm_sq = _mm512_fmadd_ps(w_x_i_vec, w_x_i_vec, _mm512_mul_ps(w_y_i_vec, w_y_i_vec)); + __m512 w_norm_inv = _mm512_rsqrt14_ps(w_norm_sq); + __m512 correction_factor = _mm512_min_ps(_mm512_set1_ps(1.0f), _mm512_mul_ps(v_max_i_vec, w_norm_inv)); + + __m512 curr_vel_x_i_vec = _mm512_mul_ps(w_x_i_vec, correction_factor); + __m512 curr_vel_y_i_vec = _mm512_mul_ps(w_y_i_vec, correction_factor); + + __m512 dt_vec = _mm512_set1_ps(dt); + pos_x_i_vec = _mm512_fmadd_ps(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); + pos_y_i_vec = _mm512_fmadd_ps(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); + + __mmask16 cond_1_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_setzero_si512()); + __mmask16 cond_1_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.5f), _CMP_GT_OQ); + __mmask16 cond_1 = cond_1_left & cond_1_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_1, pos_x_i_vec, _mm512_set1_ps(1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_1, destination_y_i_vec, pos_y_i_vec); + w_x_i_vec = _mm512_mask_blend_ps(cond_1, w_x_i_vec, desired_v0_i_vec); + w_y_i_vec = _mm512_mask_blend_ps(cond_1, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_y_i_vec, w_y_i_vec); + + __mmask16 cond_2_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_set1_epi32(1)); + __mmask16 cond_2_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(1.5f), _CMP_LT_OQ); + __mmask16 cond_2 = cond_2_left & cond_2_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_2, pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_2, destination_y_i_vec, pos_y_i_vec); + __m512 neg_desired_v0 = _mm512_xor_ps(desired_v0_i_vec, _mm512_set1_ps(-0.0f)); + w_x_i_vec = _mm512_mask_blend_ps(cond_2, w_x_i_vec, neg_desired_v0); + w_y_i_vec = _mm512_mask_blend_ps(cond_2, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_y_i_vec, w_y_i_vec); + + pos_y_i_vec = _mm512_max_ps(pos_y_i_vec, _mm512_set1_ps(WALL_EPS)); + pos_y_i_vec = _mm512_min_ps(pos_y_i_vec, _mm512_set1_ps(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512 ex_vec = _mm512_sub_ps(destination_x_i_vec, pos_x_i_vec); + __m512 ey_vec = _mm512_sub_ps(destination_y_i_vec, pos_y_i_vec); + __m512 e_norm_sq = _mm512_fmadd_ps(ex_vec, ex_vec, _mm512_mul_ps(ey_vec, ey_vec)); + __m512 e_norm_inv = _mm512_rsqrt14_ps(e_norm_sq); + __m512 e_x_i_vec = _mm512_mul_ps(ex_vec, e_norm_inv); + __m512 e_y_i_vec = _mm512_mul_ps(ey_vec, e_norm_inv); + + __m512 vel_norm_sq = _mm512_fmadd_ps(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_ps(curr_vel_y_i_vec, curr_vel_y_i_vec)); + __m512 sb_sq_i_vec = _mm512_mul_ps(vel_norm_sq, _mm512_set1_ps(delta_t_sq)); + __m512 vel_norm_dt = _mm512_sqrt_ps(sb_sq_i_vec); + + _mm512_storeu_ps(&curr_vel_x[i], curr_vel_x_i_vec); + _mm512_storeu_ps(&curr_vel_y[i], curr_vel_y_i_vec); + _mm512_storeu_ps(&w_x[i], w_x_i_vec); + _mm512_storeu_ps(&w_y[i], w_y_i_vec); + _mm512_storeu_ps(&pos_x[i], pos_x_i_vec); + _mm512_storeu_ps(&pos_y[i], pos_y_i_vec); + _mm512_storeu_ps(&e_x[i], e_x_i_vec); + _mm512_storeu_ps(&e_y[i], e_y_i_vec); + _mm512_storeu_ps(&destination_y[i], destination_y_i_vec); + _mm512_storeu_ps(&sb_sq[i], sb_sq_i_vec); + _mm512_storeu_ps(&sb_e_x[i], _mm512_mul_ps(e_x_i_vec, vel_norm_dt)); + _mm512_storeu_ps(&sb_e_y[i], _mm512_mul_ps(e_y_i_vec, vel_norm_dt)); + } + +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_4.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_4.cpp new file mode 100644 index 0000000..17095fa --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_4.cpp @@ -0,0 +1,608 @@ +// # Changes: Added ILP with k = 4 to j-loop +// # Predecessor: variant_float_inlining_SoA_AVX512_polyexp_moreavx_ilp_2.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 63) & ~63; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x_0 = _mm512_setzero_ps(); + __m512 acc_x_1 = _mm512_setzero_ps(); + __m512 acc_x_2 = _mm512_setzero_ps(); + __m512 acc_x_3 = _mm512_setzero_ps(); + + __m512 acc_y_0 = _mm512_setzero_ps(); + __m512 acc_y_1 = _mm512_setzero_ps(); + __m512 acc_y_2 = _mm512_setzero_ps(); + __m512 acc_y_3 = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 64) { + __m512 j_idx_0 = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + __m512 j_idx_1 = _mm512_add_ps(_mm512_set1_ps((float)j + 16), lane_off); + __m512 j_idx_2 = _mm512_add_ps(_mm512_set1_ps((float)j + 32), lane_off); + __m512 j_idx_3 = _mm512_add_ps(_mm512_set1_ps((float)j + 48), lane_off); + + __mmask16 m_neq_i_0 = _mm512_cmp_ps_mask(j_idx_0, v_i, _CMP_NEQ_OQ); + __mmask16 m_neq_i_1 = _mm512_cmp_ps_mask(j_idx_1, v_i, _CMP_NEQ_OQ); + __mmask16 m_neq_i_2 = _mm512_cmp_ps_mask(j_idx_2, v_i, _CMP_NEQ_OQ); + __mmask16 m_neq_i_3 = _mm512_cmp_ps_mask(j_idx_3, v_i, _CMP_NEQ_OQ); + + __mmask16 m_in_range_0 = _mm512_cmp_ps_mask(j_idx_0, v_np, _CMP_LT_OQ); + __mmask16 m_in_range_1 = _mm512_cmp_ps_mask(j_idx_1, v_np, _CMP_LT_OQ); + __mmask16 m_in_range_2 = _mm512_cmp_ps_mask(j_idx_2, v_np, _CMP_LT_OQ); + __mmask16 m_in_range_3 = _mm512_cmp_ps_mask(j_idx_3, v_np, _CMP_LT_OQ); + + __mmask16 active_0 = m_neq_i_0 & m_in_range_0; + __mmask16 active_1 = m_neq_i_1 & m_in_range_1; + __mmask16 active_2 = m_neq_i_2 & m_in_range_2; + __mmask16 active_3 = m_neq_i_3 & m_in_range_3; + + __m512 pjx_0 = _mm512_loadu_ps(&pos_x[j]); + __m512 pjx_1 = _mm512_loadu_ps(&pos_x[j + 16]); + __m512 pjx_2 = _mm512_loadu_ps(&pos_x[j + 32]); + __m512 pjx_3 = _mm512_loadu_ps(&pos_x[j + 48]); + + __m512 pjy_0 = _mm512_loadu_ps(&pos_y[j]); + __m512 pjy_1 = _mm512_loadu_ps(&pos_y[j + 16]); + __m512 pjy_2 = _mm512_loadu_ps(&pos_y[j + 32]); + __m512 pjy_3 = _mm512_loadu_ps(&pos_y[j + 48]); + + __m512 sbe_jx_0 = _mm512_loadu_ps(&sb_e_x[j]); + __m512 sbe_jx_1 = _mm512_loadu_ps(&sb_e_x[j + 16]); + __m512 sbe_jx_2 = _mm512_loadu_ps(&sb_e_x[j + 32]); + __m512 sbe_jx_3 = _mm512_loadu_ps(&sb_e_x[j + 48]); + + __m512 sbe_jy_0 = _mm512_loadu_ps(&sb_e_y[j]); + __m512 sbe_jy_1 = _mm512_loadu_ps(&sb_e_y[j + 16]); + __m512 sbe_jy_2 = _mm512_loadu_ps(&sb_e_y[j + 32]); + __m512 sbe_jy_3 = _mm512_loadu_ps(&sb_e_y[j + 48]); + + __m512 sb_jsq_0 = _mm512_loadu_ps(&sb_sq[j]); + __m512 sb_jsq_1 = _mm512_loadu_ps(&sb_sq[j + 16]); + __m512 sb_jsq_2 = _mm512_loadu_ps(&sb_sq[j + 32]); + __m512 sb_jsq_3 = _mm512_loadu_ps(&sb_sq[j + 48]); + + __m512 r_ab_x_0 = _mm512_sub_ps(pix, pjx_0); + __m512 r_ab_x_1 = _mm512_sub_ps(pix, pjx_1); + __m512 r_ab_x_2 = _mm512_sub_ps(pix, pjx_2); + __m512 r_ab_x_3 = _mm512_sub_ps(pix, pjx_3); + + __m512 r_ab_y_0 = _mm512_sub_ps(piy, pjy_0); + __m512 r_ab_y_1 = _mm512_sub_ps(piy, pjy_1); + __m512 r_ab_y_2 = _mm512_sub_ps(piy, pjy_2); + __m512 r_ab_y_3 = _mm512_sub_ps(piy, pjy_3); + + __m512 r_ab_n2_0 = _mm512_fmadd_ps(r_ab_x_0, r_ab_x_0, _mm512_mul_ps(r_ab_y_0, r_ab_y_0)); + __m512 r_ab_n2_1 = _mm512_fmadd_ps(r_ab_x_1, r_ab_x_1, _mm512_mul_ps(r_ab_y_1, r_ab_y_1)); + __m512 r_ab_n2_2 = _mm512_fmadd_ps(r_ab_x_2, r_ab_x_2, _mm512_mul_ps(r_ab_y_2, r_ab_y_2)); + __m512 r_ab_n2_3 = _mm512_fmadd_ps(r_ab_x_3, r_ab_x_3, _mm512_mul_ps(r_ab_y_3, r_ab_y_3)); + + __m512 r_ab_n_0 = _mm512_sqrt_ps(r_ab_n2_0); + __m512 r_ab_n_1 = _mm512_sqrt_ps(r_ab_n2_1); + __m512 r_ab_n_2 = _mm512_sqrt_ps(r_ab_n2_2); + __m512 r_ab_n_3 = _mm512_sqrt_ps(r_ab_n2_3); + + __m512 r_ab2_x_0 = _mm512_sub_ps(r_ab_x_0, sbe_jx_0); + __m512 r_ab2_x_1 = _mm512_sub_ps(r_ab_x_1, sbe_jx_1); + __m512 r_ab2_x_2 = _mm512_sub_ps(r_ab_x_2, sbe_jx_2); + __m512 r_ab2_x_3 = _mm512_sub_ps(r_ab_x_3, sbe_jx_3); + + __m512 r_ab2_y_0 = _mm512_sub_ps(r_ab_y_0, sbe_jy_0); + __m512 r_ab2_y_1 = _mm512_sub_ps(r_ab_y_1, sbe_jy_1); + __m512 r_ab2_y_2 = _mm512_sub_ps(r_ab_y_2, sbe_jy_2); + __m512 r_ab2_y_3 = _mm512_sub_ps(r_ab_y_3, sbe_jy_3); + + __m512 r_ab2_n2_0 = _mm512_fmadd_ps(r_ab2_x_0, r_ab2_x_0, _mm512_mul_ps(r_ab2_y_0, r_ab2_y_0)); + __m512 r_ab2_n2_1 = _mm512_fmadd_ps(r_ab2_x_1, r_ab2_x_1, _mm512_mul_ps(r_ab2_y_1, r_ab2_y_1)); + __m512 r_ab2_n2_2 = _mm512_fmadd_ps(r_ab2_x_2, r_ab2_x_2, _mm512_mul_ps(r_ab2_y_2, r_ab2_y_2)); + __m512 r_ab2_n2_3 = _mm512_fmadd_ps(r_ab2_x_3, r_ab2_x_3, _mm512_mul_ps(r_ab2_y_3, r_ab2_y_3)); + + __m512 r_ab2_n_0 = _mm512_sqrt_ps(r_ab2_n2_0); + __m512 r_ab2_n_1 = _mm512_sqrt_ps(r_ab2_n2_1); + __m512 r_ab2_n_2 = _mm512_sqrt_ps(r_ab2_n2_2); + __m512 r_ab2_n_3 = _mm512_sqrt_ps(r_ab2_n2_3); + + __m512 r_sum_0 = _mm512_add_ps(r_ab_n_0, r_ab2_n_0); + __m512 r_sum_1 = _mm512_add_ps(r_ab_n_1, r_ab2_n_1); + __m512 r_sum_2 = _mm512_add_ps(r_ab_n_2, r_ab2_n_2); + __m512 r_sum_3 = _mm512_add_ps(r_ab_n_3, r_ab2_n_3); + + __m512 two_b_sq_0 = _mm512_fmsub_ps(r_sum_0, r_sum_0, sb_jsq_0); + __m512 two_b_sq_1 = _mm512_fmsub_ps(r_sum_1, r_sum_1, sb_jsq_1); + __m512 two_b_sq_2 = _mm512_fmsub_ps(r_sum_2, r_sum_2, sb_jsq_2); + __m512 two_b_sq_3 = _mm512_fmsub_ps(r_sum_3, r_sum_3, sb_jsq_3); + + __m512 two_b_0 = _mm512_sqrt_ps(two_b_sq_0); + __m512 two_b_1 = _mm512_sqrt_ps(two_b_sq_1); + __m512 two_b_2 = _mm512_sqrt_ps(two_b_sq_2); + __m512 two_b_3 = _mm512_sqrt_ps(two_b_sq_3); + + __m512 e_arg_0 = _mm512_mul_ps(two_b_0, v_neg_2s_inv); + __m512 e_arg_1 = _mm512_mul_ps(two_b_1, v_neg_2s_inv); + __m512 e_arg_2 = _mm512_mul_ps(two_b_2, v_neg_2s_inv); + __m512 e_arg_3 = _mm512_mul_ps(two_b_3, v_neg_2s_inv); + + e_arg_0 = _mm512_max_ps(e_arg_0, v_exp_min); + e_arg_1 = _mm512_max_ps(e_arg_1, v_exp_min); + e_arg_2 = _mm512_max_ps(e_arg_2, v_exp_min); + e_arg_3 = _mm512_max_ps(e_arg_3, v_exp_min); + + __m512 e_val_0 = _avx512_exp_ps(e_arg_0); + __m512 e_val_1 = _avx512_exp_ps(e_arg_1); + __m512 e_val_2 = _avx512_exp_ps(e_arg_2); + __m512 e_val_3 = _avx512_exp_ps(e_arg_3); + + __m512 numer_0 = _mm512_mul_ps(_mm512_mul_ps(r_sum_0, v_neg_v0_div_2s), e_val_0); + __m512 numer_1 = _mm512_mul_ps(_mm512_mul_ps(r_sum_1, v_neg_v0_div_2s), e_val_1); + __m512 numer_2 = _mm512_mul_ps(_mm512_mul_ps(r_sum_2, v_neg_v0_div_2s), e_val_2); + __m512 numer_3 = _mm512_mul_ps(_mm512_mul_ps(r_sum_3, v_neg_v0_div_2s), e_val_3); + + __m512 denom_0 = _mm512_mul_ps(_mm512_mul_ps(two_b_0, r_ab_n_0), r_ab2_n_0); + __m512 denom_1 = _mm512_mul_ps(_mm512_mul_ps(two_b_1, r_ab_n_1), r_ab2_n_1); + __m512 denom_2 = _mm512_mul_ps(_mm512_mul_ps(two_b_2, r_ab_n_2), r_ab2_n_2); + __m512 denom_3 = _mm512_mul_ps(_mm512_mul_ps(two_b_3, r_ab_n_3), r_ab2_n_3); + + __m512 denom_safe_0 = _mm512_mask_blend_ps(active_0, v_one, denom_0); + __m512 denom_safe_1 = _mm512_mask_blend_ps(active_1, v_one, denom_1); + __m512 denom_safe_2 = _mm512_mask_blend_ps(active_2, v_one, denom_2); + __m512 denom_safe_3 = _mm512_mask_blend_ps(active_3, v_one, denom_3); + + __m512 smc_0 = _mm512_div_ps(numer_0, denom_safe_0); + __m512 smc_1 = _mm512_div_ps(numer_1, denom_safe_1); + __m512 smc_2 = _mm512_div_ps(numer_2, denom_safe_2); + __m512 smc_3 = _mm512_div_ps(numer_3, denom_safe_3); + + __m512 sm_r_ab_0 = _mm512_mul_ps(smc_0, r_ab2_n_0); + __m512 sm_r_ab_1 = _mm512_mul_ps(smc_1, r_ab2_n_1); + __m512 sm_r_ab_2 = _mm512_mul_ps(smc_2, r_ab2_n_2); + __m512 sm_r_ab_3 = _mm512_mul_ps(smc_3, r_ab2_n_3); + + __m512 sm_r_ab2_0 = _mm512_mul_ps(smc_0, r_ab_n_0); + __m512 sm_r_ab2_1 = _mm512_mul_ps(smc_1, r_ab_n_1); + __m512 sm_r_ab2_2 = _mm512_mul_ps(smc_2, r_ab_n_2); + __m512 sm_r_ab2_3 = _mm512_mul_ps(smc_3, r_ab_n_3); + + __m512 f_x_0 = _mm512_fmadd_ps(r_ab2_x_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_x_0, sm_r_ab_0)); + __m512 f_x_1 = _mm512_fmadd_ps(r_ab2_x_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_x_1, sm_r_ab_1)); + __m512 f_x_2 = _mm512_fmadd_ps(r_ab2_x_2, sm_r_ab2_2, _mm512_mul_ps(r_ab_x_2, sm_r_ab_2)); + __m512 f_x_3 = _mm512_fmadd_ps(r_ab2_x_3, sm_r_ab2_3, _mm512_mul_ps(r_ab_x_3, sm_r_ab_3)); + + __m512 f_y_0 = _mm512_fmadd_ps(r_ab2_y_0, sm_r_ab2_0, _mm512_mul_ps(r_ab_y_0, sm_r_ab_0)); + __m512 f_y_1 = _mm512_fmadd_ps(r_ab2_y_1, sm_r_ab2_1, _mm512_mul_ps(r_ab_y_1, sm_r_ab_1)); + __m512 f_y_2 = _mm512_fmadd_ps(r_ab2_y_2, sm_r_ab2_2, _mm512_mul_ps(r_ab_y_2, sm_r_ab_2)); + __m512 f_y_3 = _mm512_fmadd_ps(r_ab2_y_3, sm_r_ab2_3, _mm512_mul_ps(r_ab_y_3, sm_r_ab_3)); + + __m512 f_norm_sq_0 = _mm512_fmadd_ps(f_x_0, f_x_0, _mm512_mul_ps(f_y_0, f_y_0)); + __m512 f_norm_sq_1 = _mm512_fmadd_ps(f_x_1, f_x_1, _mm512_mul_ps(f_y_1, f_y_1)); + __m512 f_norm_sq_2 = _mm512_fmadd_ps(f_x_2, f_x_2, _mm512_mul_ps(f_y_2, f_y_2)); + __m512 f_norm_sq_3 = _mm512_fmadd_ps(f_x_3, f_x_3, _mm512_mul_ps(f_y_3, f_y_3)); + + __m512 rhs_0 = _mm512_mul_ps(f_norm_sq_0, v_cos_phi_sq); + __m512 rhs_1 = _mm512_mul_ps(f_norm_sq_1, v_cos_phi_sq); + __m512 rhs_2 = _mm512_mul_ps(f_norm_sq_2, v_cos_phi_sq); + __m512 rhs_3 = _mm512_mul_ps(f_norm_sq_3, v_cos_phi_sq); + + __m512 e_dot_f_0 = _mm512_fmadd_ps(eix, f_x_0, _mm512_mul_ps(eiy, f_y_0)); + __m512 e_dot_f_1 = _mm512_fmadd_ps(eix, f_x_1, _mm512_mul_ps(eiy, f_y_1)); + __m512 e_dot_f_2 = _mm512_fmadd_ps(eix, f_x_2, _mm512_mul_ps(eiy, f_y_2)); + __m512 e_dot_f_3 = _mm512_fmadd_ps(eix, f_x_3, _mm512_mul_ps(eiy, f_y_3)); + + __m512 e_dot_f_sq_0 = _mm512_mul_ps(e_dot_f_0, e_dot_f_0); + __m512 e_dot_f_sq_1 = _mm512_mul_ps(e_dot_f_1, e_dot_f_1); + __m512 e_dot_f_sq_2 = _mm512_mul_ps(e_dot_f_2, e_dot_f_2); + __m512 e_dot_f_sq_3 = _mm512_mul_ps(e_dot_f_3, e_dot_f_3); + + __mmask16 c1_0 = _mm512_cmp_ps_mask(e_dot_f_0, v_zero, _CMP_LT_OQ); + __mmask16 c1_1 = _mm512_cmp_ps_mask(e_dot_f_1, v_zero, _CMP_LT_OQ); + __mmask16 c1_2 = _mm512_cmp_ps_mask(e_dot_f_2, v_zero, _CMP_LT_OQ); + __mmask16 c1_3 = _mm512_cmp_ps_mask(e_dot_f_3, v_zero, _CMP_LT_OQ); + + __mmask16 c2_0 = _mm512_cmp_ps_mask(e_dot_f_sq_0, rhs_0, _CMP_GT_OQ); + __mmask16 c2_1 = _mm512_cmp_ps_mask(e_dot_f_sq_1, rhs_1, _CMP_GT_OQ); + __mmask16 c2_2 = _mm512_cmp_ps_mask(e_dot_f_sq_2, rhs_2, _CMP_GT_OQ); + __mmask16 c2_3 = _mm512_cmp_ps_mask(e_dot_f_sq_3, rhs_3, _CMP_GT_OQ); + + __mmask16 cond_0 = c1_0 & c2_0; + __mmask16 cond_1 = c1_1 & c2_1; + __mmask16 cond_2 = c1_2 & c2_2; + __mmask16 cond_3 = c1_3 & c2_3; + + __m512 w_val_0 = _mm512_mask_blend_ps(cond_0, v_dt_neg, v_c_neg_dt); + __m512 w_val_1 = _mm512_mask_blend_ps(cond_1, v_dt_neg, v_c_neg_dt); + __m512 w_val_2 = _mm512_mask_blend_ps(cond_2, v_dt_neg, v_c_neg_dt); + __m512 w_val_3 = _mm512_mask_blend_ps(cond_3, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe_0 = _mm512_maskz_mov_ps(active_0, f_x_0); + __m512 f_x_safe_1 = _mm512_maskz_mov_ps(active_1, f_x_1); + __m512 f_x_safe_2 = _mm512_maskz_mov_ps(active_2, f_x_2); + __m512 f_x_safe_3 = _mm512_maskz_mov_ps(active_3, f_x_3); + + __m512 f_y_safe_0 = _mm512_maskz_mov_ps(active_0, f_y_0); + __m512 f_y_safe_1 = _mm512_maskz_mov_ps(active_1, f_y_1); + __m512 f_y_safe_2 = _mm512_maskz_mov_ps(active_2, f_y_2); + __m512 f_y_safe_3 = _mm512_maskz_mov_ps(active_3, f_y_3); + + acc_x_0 = _mm512_fmadd_ps(f_x_safe_0, w_val_0, acc_x_0); + acc_x_1 = _mm512_fmadd_ps(f_x_safe_1, w_val_1, acc_x_1); + acc_x_2 = _mm512_fmadd_ps(f_x_safe_2, w_val_2, acc_x_2); + acc_x_3 = _mm512_fmadd_ps(f_x_safe_3, w_val_3, acc_x_3); + + acc_y_0 = _mm512_fmadd_ps(f_y_safe_0, w_val_0, acc_y_0); + acc_y_1 = _mm512_fmadd_ps(f_y_safe_1, w_val_1, acc_y_1); + acc_y_2 = _mm512_fmadd_ps(f_y_safe_2, w_val_2, acc_y_2); + acc_y_3 = _mm512_fmadd_ps(f_y_safe_3, w_val_3, acc_y_3); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(acc_x_0, acc_x_1), _mm512_add_ps(acc_x_2, acc_x_3))); + f_alpha_dt_y += _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(acc_y_0, acc_y_1), _mm512_add_ps(acc_y_2, acc_y_3))); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 16) { + + __m512 w_x_i_vec = _mm512_loadu_ps(&w_x[i]); + __m512 w_y_i_vec = _mm512_loadu_ps(&w_y[i]); + __m512 v_max_i_vec = _mm512_loadu_ps(&v_max[i]); + __m512 pos_x_i_vec = _mm512_loadu_ps(&pos_x[i]); + __m512 pos_y_i_vec = _mm512_loadu_ps(&pos_y[i]); + __m512 destination_y_i_vec = _mm512_loadu_ps(&destination_y[i]); + __m512 destination_x_i_vec = _mm512_loadu_ps(&destination_x[i]); + __m512 desired_v0_i_vec = _mm512_loadu_ps(&desired_v0[i]); + + __m512i color_i_vec = _mm512_loadu_si512((__m512i*)&color[i]); + + __m512 w_norm_sq = _mm512_fmadd_ps(w_x_i_vec, w_x_i_vec, _mm512_mul_ps(w_y_i_vec, w_y_i_vec)); + __m512 w_norm = _mm512_sqrt_ps(w_norm_sq); + __m512 correction_factor = _mm512_min_ps(_mm512_set1_ps(1.0f), _mm512_div_ps(v_max_i_vec, w_norm)); + + __m512 curr_vel_x_i_vec = _mm512_mul_ps(w_x_i_vec, correction_factor); + __m512 curr_vel_y_i_vec = _mm512_mul_ps(w_y_i_vec, correction_factor); + + __m512 dt_vec = _mm512_set1_ps(dt); + pos_x_i_vec = _mm512_fmadd_ps(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); + pos_y_i_vec = _mm512_fmadd_ps(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); + + __mmask16 cond_1_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_setzero_si512()); + __mmask16 cond_1_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.5f), _CMP_GT_OQ); + __mmask16 cond_1 = cond_1_left & cond_1_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_1, pos_x_i_vec, _mm512_set1_ps(1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_1, destination_y_i_vec, pos_y_i_vec); + w_x_i_vec = _mm512_mask_blend_ps(cond_1, w_x_i_vec, desired_v0_i_vec); + w_y_i_vec = _mm512_mask_blend_ps(cond_1, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_y_i_vec, w_y_i_vec); + + __mmask16 cond_2_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_set1_epi32(1)); + __mmask16 cond_2_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(1.5f), _CMP_LT_OQ); + __mmask16 cond_2 = cond_2_left & cond_2_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_2, pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_2, destination_y_i_vec, pos_y_i_vec); + __m512 neg_desired_v0 = _mm512_xor_ps(desired_v0_i_vec, _mm512_set1_ps(-0.0f)); + w_x_i_vec = _mm512_mask_blend_ps(cond_2, w_x_i_vec, neg_desired_v0); + w_y_i_vec = _mm512_mask_blend_ps(cond_2, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_y_i_vec, w_y_i_vec); + + pos_y_i_vec = _mm512_max_ps(pos_y_i_vec, _mm512_set1_ps(WALL_EPS)); + pos_y_i_vec = _mm512_min_ps(pos_y_i_vec, _mm512_set1_ps(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512 ex_vec = _mm512_sub_ps(destination_x_i_vec, pos_x_i_vec); + __m512 ey_vec = _mm512_sub_ps(destination_y_i_vec, pos_y_i_vec); + __m512 e_norm_sq = _mm512_fmadd_ps(ex_vec, ex_vec, _mm512_mul_ps(ey_vec, ey_vec)); + __m512 e_norm = _mm512_sqrt_ps(e_norm_sq); + __m512 e_norm_inv = _mm512_div_ps(_mm512_set1_ps(1.0f), e_norm); + __m512 e_x_i_vec = _mm512_mul_ps(ex_vec, e_norm_inv); + __m512 e_y_i_vec = _mm512_mul_ps(ey_vec, e_norm_inv); + + __m512 vel_norm_sq = _mm512_fmadd_ps(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_ps(curr_vel_y_i_vec, curr_vel_y_i_vec)); + __m512 sb_sq_i_vec = _mm512_mul_ps(vel_norm_sq, _mm512_set1_ps(delta_t_sq)); + __m512 vel_norm_dt = _mm512_sqrt_ps(sb_sq_i_vec); + + _mm512_storeu_ps(&curr_vel_x[i], curr_vel_x_i_vec); + _mm512_storeu_ps(&curr_vel_y[i], curr_vel_y_i_vec); + _mm512_storeu_ps(&w_x[i], w_x_i_vec); + _mm512_storeu_ps(&w_y[i], w_y_i_vec); + _mm512_storeu_ps(&pos_x[i], pos_x_i_vec); + _mm512_storeu_ps(&pos_y[i], pos_y_i_vec); + _mm512_storeu_ps(&e_x[i], e_x_i_vec); + _mm512_storeu_ps(&e_y[i], e_y_i_vec); + _mm512_storeu_ps(&destination_y[i], destination_y_i_vec); + _mm512_storeu_ps(&sb_sq[i], sb_sq_i_vec); + _mm512_storeu_ps(&sb_e_x[i], _mm512_mul_ps(e_x_i_vec, vel_norm_dt)); + _mm512_storeu_ps(&sb_e_y[i], _mm512_mul_ps(e_y_i_vec, vel_norm_dt)); + } + + +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_symmetry_ILP_16.cpp b/variants/variant_float_inlining_SoA_symmetry_ILP_16.cpp new file mode 100644 index 0000000..8e5cd0c --- /dev/null +++ b/variants/variant_float_inlining_SoA_symmetry_ILP_16.cpp @@ -0,0 +1,1365 @@ +// # Changes: Added ILP for j-loop with K = 16 +// # Predecessor: variant_float_inlining_SoA_symmetry_ILP_8.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +vector pos_x, pos_y; +vector sb_sq; +vector sb_e_x, sb_e_y; +vector e_x, e_y; +vector curr_vel_x, curr_vel_y; +vector w_x, w_y; +vector desired_v0; +vector v_max; +vector destination_x, destination_y; +vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + pos_x.resize(n_p); + pos_y.resize(n_p); + sb_sq.resize(n_p); + sb_e_x.resize(n_p); + sb_e_y.resize(n_p); + e_x.resize(n_p); + e_y.resize(n_p); + curr_vel_x.resize(n_p); + curr_vel_y.resize(n_p); + w_x.resize(n_p); + w_y.resize(n_p); + desired_v0.resize(n_p); + v_max.resize(n_p); + destination_x.resize(n_p); + destination_y.resize(n_p); + color.resize(n_p); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + vector r_ab_x_lut(n_p * n_p, 0.0); + vector r_ab_y_lut(n_p * n_p, 0.0); + vector r_ab_norm_lut(n_p * n_p, 0.0); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + float e_x_i = e_x[i]; + float e_y_i = e_y[i]; + float pos_x_i = pos_x[i]; + float pos_y_i = pos_y[i]; + + for (int j = i + 1; j < n_p; j++) { + float dx = pos_x_i - pos_x[j]; + float dy = pos_y_i - pos_y[j]; + float norm = sqrt(dx * dx + dy * dy); + int idx = i * n_p + j; + r_ab_x_lut[idx] = dx; + r_ab_y_lut[idx] = dy; + r_ab_norm_lut[idx] = norm; + } + + int j; + + float acc_x_0 = 0.0f, acc_x_1 = 0.0f, acc_x_2 = 0.0f, acc_x_3 = 0.0f; + float acc_x_4 = 0.0f, acc_x_5 = 0.0f, acc_x_6 = 0.0f, acc_x_7 = 0.0f; + float acc_x_8 = 0.0f, acc_x_9 = 0.0f, acc_x_10 = 0.0f, acc_x_11 = 0.0f; + float acc_x_12 = 0.0f, acc_x_13 = 0.0f, acc_x_14 = 0.0f, acc_x_15 = 0.0f; + float acc_y_0 = 0.0f, acc_y_1 = 0.0f, acc_y_2 = 0.0f, acc_y_3 = 0.0f; + float acc_y_4 = 0.0f, acc_y_5 = 0.0f, acc_y_6 = 0.0f, acc_y_7 = 0.0f; + float acc_y_8 = 0.0f, acc_y_9 = 0.0f, acc_y_10 = 0.0f, acc_y_11 = 0.0f; + float acc_y_12 = 0.0f, acc_y_13 = 0.0f, acc_y_14 = 0.0f, acc_y_15 = 0.0f; + + for (j = 0; j + 16 < i; j += 16) { + float r_ab_x_0 = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_x_1 = -1 * r_ab_x_lut[(j + 1) * n_p + i]; + float r_ab_x_2 = -1 * r_ab_x_lut[(j + 2) * n_p + i]; + float r_ab_x_3 = -1 * r_ab_x_lut[(j + 3) * n_p + i]; + float r_ab_x_4 = -1 * r_ab_x_lut[(j + 4) * n_p + i]; + float r_ab_x_5 = -1 * r_ab_x_lut[(j + 5) * n_p + i]; + float r_ab_x_6 = -1 * r_ab_x_lut[(j + 6) * n_p + i]; + float r_ab_x_7 = -1 * r_ab_x_lut[(j + 7) * n_p + i]; + float r_ab_x_8 = -1 * r_ab_x_lut[(j + 8) * n_p + i]; + float r_ab_x_9 = -1 * r_ab_x_lut[(j + 9) * n_p + i]; + float r_ab_x_10 = -1 * r_ab_x_lut[(j + 10) * n_p + i]; + float r_ab_x_11 = -1 * r_ab_x_lut[(j + 11) * n_p + i]; + float r_ab_x_12 = -1 * r_ab_x_lut[(j + 12) * n_p + i]; + float r_ab_x_13 = -1 * r_ab_x_lut[(j + 13) * n_p + i]; + float r_ab_x_14 = -1 * r_ab_x_lut[(j + 14) * n_p + i]; + float r_ab_x_15 = -1 * r_ab_x_lut[(j + 15) * n_p + i]; + + float r_ab_y_0 = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_y_1 = -1 * r_ab_y_lut[(j + 1) * n_p + i]; + float r_ab_y_2 = -1 * r_ab_y_lut[(j + 2) * n_p + i]; + float r_ab_y_3 = -1 * r_ab_y_lut[(j + 3) * n_p + i]; + float r_ab_y_4 = -1 * r_ab_y_lut[(j + 4) * n_p + i]; + float r_ab_y_5 = -1 * r_ab_y_lut[(j + 5) * n_p + i]; + float r_ab_y_6 = -1 * r_ab_y_lut[(j + 6) * n_p + i]; + float r_ab_y_7 = -1 * r_ab_y_lut[(j + 7) * n_p + i]; + float r_ab_y_8 = -1 * r_ab_y_lut[(j + 8) * n_p + i]; + float r_ab_y_9 = -1 * r_ab_y_lut[(j + 9) * n_p + i]; + float r_ab_y_10 = -1 * r_ab_y_lut[(j + 10) * n_p + i]; + float r_ab_y_11 = -1 * r_ab_y_lut[(j + 11) * n_p + i]; + float r_ab_y_12 = -1 * r_ab_y_lut[(j + 12) * n_p + i]; + float r_ab_y_13 = -1 * r_ab_y_lut[(j + 13) * n_p + i]; + float r_ab_y_14 = -1 * r_ab_y_lut[(j + 14) * n_p + i]; + float r_ab_y_15 = -1 * r_ab_y_lut[(j + 15) * n_p + i]; + + float r_ab_norm_0 = r_ab_norm_lut[j * n_p + i]; + float r_ab_norm_1 = r_ab_norm_lut[(j + 1) * n_p + i]; + float r_ab_norm_2 = r_ab_norm_lut[(j + 2) * n_p + i]; + float r_ab_norm_3 = r_ab_norm_lut[(j + 3) * n_p + i]; + float r_ab_norm_4 = r_ab_norm_lut[(j + 4) * n_p + i]; + float r_ab_norm_5 = r_ab_norm_lut[(j + 5) * n_p + i]; + float r_ab_norm_6 = r_ab_norm_lut[(j + 6) * n_p + i]; + float r_ab_norm_7 = r_ab_norm_lut[(j + 7) * n_p + i]; + float r_ab_norm_8 = r_ab_norm_lut[(j + 8) * n_p + i]; + float r_ab_norm_9 = r_ab_norm_lut[(j + 9) * n_p + i]; + float r_ab_norm_10 = r_ab_norm_lut[(j + 10) * n_p + i]; + float r_ab_norm_11 = r_ab_norm_lut[(j + 11) * n_p + i]; + float r_ab_norm_12 = r_ab_norm_lut[(j + 12) * n_p + i]; + float r_ab_norm_13 = r_ab_norm_lut[(j + 13) * n_p + i]; + float r_ab_norm_14 = r_ab_norm_lut[(j + 14) * n_p + i]; + float r_ab_norm_15 = r_ab_norm_lut[(j + 15) * n_p + i]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + r_ab_norm_2 = r_ab_norm_2 > EPS ? r_ab_norm_2 : EPS; + r_ab_norm_3 = r_ab_norm_3 > EPS ? r_ab_norm_3 : EPS; + r_ab_norm_4 = r_ab_norm_4 > EPS ? r_ab_norm_4 : EPS; + r_ab_norm_5 = r_ab_norm_5 > EPS ? r_ab_norm_5 : EPS; + r_ab_norm_6 = r_ab_norm_6 > EPS ? r_ab_norm_6 : EPS; + r_ab_norm_7 = r_ab_norm_7 > EPS ? r_ab_norm_7 : EPS; + r_ab_norm_8 = r_ab_norm_8 > EPS ? r_ab_norm_8 : EPS; + r_ab_norm_9 = r_ab_norm_9 > EPS ? r_ab_norm_9 : EPS; + r_ab_norm_10 = r_ab_norm_10 > EPS ? r_ab_norm_10 : EPS; + r_ab_norm_11 = r_ab_norm_11 > EPS ? r_ab_norm_11 : EPS; + r_ab_norm_12 = r_ab_norm_12 > EPS ? r_ab_norm_12 : EPS; + r_ab_norm_13 = r_ab_norm_13 > EPS ? r_ab_norm_13 : EPS; + r_ab_norm_14 = r_ab_norm_14 > EPS ? r_ab_norm_14 : EPS; + r_ab_norm_15 = r_ab_norm_15 > EPS ? r_ab_norm_15 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + float r_ab2_x_2 = r_ab_x_2 - sb_e_x[j + 2]; + float r_ab2_x_3 = r_ab_x_3 - sb_e_x[j + 3]; + float r_ab2_x_4 = r_ab_x_4 - sb_e_x[j + 4]; + float r_ab2_x_5 = r_ab_x_5 - sb_e_x[j + 5]; + float r_ab2_x_6 = r_ab_x_6 - sb_e_x[j + 6]; + float r_ab2_x_7 = r_ab_x_7 - sb_e_x[j + 7]; + float r_ab2_x_8 = r_ab_x_8 - sb_e_x[j + 8]; + float r_ab2_x_9 = r_ab_x_9 - sb_e_x[j + 9]; + float r_ab2_x_10 = r_ab_x_10 - sb_e_x[j + 10]; + float r_ab2_x_11 = r_ab_x_11 - sb_e_x[j + 11]; + float r_ab2_x_12 = r_ab_x_12 - sb_e_x[j + 12]; + float r_ab2_x_13 = r_ab_x_13 - sb_e_x[j + 13]; + float r_ab2_x_14 = r_ab_x_14 - sb_e_x[j + 14]; + float r_ab2_x_15 = r_ab_x_15 - sb_e_x[j + 15]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + float r_ab2_y_2 = r_ab_y_2 - sb_e_y[j + 2]; + float r_ab2_y_3 = r_ab_y_3 - sb_e_y[j + 3]; + float r_ab2_y_4 = r_ab_y_4 - sb_e_y[j + 4]; + float r_ab2_y_5 = r_ab_y_5 - sb_e_y[j + 5]; + float r_ab2_y_6 = r_ab_y_6 - sb_e_y[j + 6]; + float r_ab2_y_7 = r_ab_y_7 - sb_e_y[j + 7]; + float r_ab2_y_8 = r_ab_y_8 - sb_e_y[j + 8]; + float r_ab2_y_9 = r_ab_y_9 - sb_e_y[j + 9]; + float r_ab2_y_10 = r_ab_y_10 - sb_e_y[j + 10]; + float r_ab2_y_11 = r_ab_y_11 - sb_e_y[j + 11]; + float r_ab2_y_12 = r_ab_y_12 - sb_e_y[j + 12]; + float r_ab2_y_13 = r_ab_y_13 - sb_e_y[j + 13]; + float r_ab2_y_14 = r_ab_y_14 - sb_e_y[j + 14]; + float r_ab2_y_15 = r_ab_y_15 - sb_e_y[j + 15]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + float r_ab2_norm_2 = sqrt(r_ab2_x_2 * r_ab2_x_2 + r_ab2_y_2 * r_ab2_y_2); + float r_ab2_norm_3 = sqrt(r_ab2_x_3 * r_ab2_x_3 + r_ab2_y_3 * r_ab2_y_3); + float r_ab2_norm_4 = sqrt(r_ab2_x_4 * r_ab2_x_4 + r_ab2_y_4 * r_ab2_y_4); + float r_ab2_norm_5 = sqrt(r_ab2_x_5 * r_ab2_x_5 + r_ab2_y_5 * r_ab2_y_5); + float r_ab2_norm_6 = sqrt(r_ab2_x_6 * r_ab2_x_6 + r_ab2_y_6 * r_ab2_y_6); + float r_ab2_norm_7 = sqrt(r_ab2_x_7 * r_ab2_x_7 + r_ab2_y_7 * r_ab2_y_7); + float r_ab2_norm_8 = sqrt(r_ab2_x_8 * r_ab2_x_8 + r_ab2_y_8 * r_ab2_y_8); + float r_ab2_norm_9 = sqrt(r_ab2_x_9 * r_ab2_x_9 + r_ab2_y_9 * r_ab2_y_9); + float r_ab2_norm_10 = sqrt(r_ab2_x_10 * r_ab2_x_10 + r_ab2_y_10 * r_ab2_y_10); + float r_ab2_norm_11 = sqrt(r_ab2_x_11 * r_ab2_x_11 + r_ab2_y_11 * r_ab2_y_11); + float r_ab2_norm_12 = sqrt(r_ab2_x_12 * r_ab2_x_12 + r_ab2_y_12 * r_ab2_y_12); + float r_ab2_norm_13 = sqrt(r_ab2_x_13 * r_ab2_x_13 + r_ab2_y_13 * r_ab2_y_13); + float r_ab2_norm_14 = sqrt(r_ab2_x_14 * r_ab2_x_14 + r_ab2_y_14 * r_ab2_y_14); + float r_ab2_norm_15 = sqrt(r_ab2_x_15 * r_ab2_x_15 + r_ab2_y_15 * r_ab2_y_15); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + r_ab2_norm_2 = r_ab2_norm_2 > EPS ? r_ab2_norm_2 : EPS; + r_ab2_norm_3 = r_ab2_norm_3 > EPS ? r_ab2_norm_3 : EPS; + r_ab2_norm_4 = r_ab2_norm_4 > EPS ? r_ab2_norm_4 : EPS; + r_ab2_norm_5 = r_ab2_norm_5 > EPS ? r_ab2_norm_5 : EPS; + r_ab2_norm_6 = r_ab2_norm_6 > EPS ? r_ab2_norm_6 : EPS; + r_ab2_norm_7 = r_ab2_norm_7 > EPS ? r_ab2_norm_7 : EPS; + r_ab2_norm_8 = r_ab2_norm_8 > EPS ? r_ab2_norm_8 : EPS; + r_ab2_norm_9 = r_ab2_norm_9 > EPS ? r_ab2_norm_9 : EPS; + r_ab2_norm_10 = r_ab2_norm_10 > EPS ? r_ab2_norm_10 : EPS; + r_ab2_norm_11 = r_ab2_norm_11 > EPS ? r_ab2_norm_11 : EPS; + r_ab2_norm_12 = r_ab2_norm_12 > EPS ? r_ab2_norm_12 : EPS; + r_ab2_norm_13 = r_ab2_norm_13 > EPS ? r_ab2_norm_13 : EPS; + r_ab2_norm_14 = r_ab2_norm_14 > EPS ? r_ab2_norm_14 : EPS; + r_ab2_norm_15 = r_ab2_norm_15 > EPS ? r_ab2_norm_15 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + float r_ab_norm_sum_2 = r_ab_norm_2 + r_ab2_norm_2; + float r_ab_norm_sum_3 = r_ab_norm_3 + r_ab2_norm_3; + float r_ab_norm_sum_4 = r_ab_norm_4 + r_ab2_norm_4; + float r_ab_norm_sum_5 = r_ab_norm_5 + r_ab2_norm_5; + float r_ab_norm_sum_6 = r_ab_norm_6 + r_ab2_norm_6; + float r_ab_norm_sum_7 = r_ab_norm_7 + r_ab2_norm_7; + float r_ab_norm_sum_8 = r_ab_norm_8 + r_ab2_norm_8; + float r_ab_norm_sum_9 = r_ab_norm_9 + r_ab2_norm_9; + float r_ab_norm_sum_10 = r_ab_norm_10 + r_ab2_norm_10; + float r_ab_norm_sum_11 = r_ab_norm_11 + r_ab2_norm_11; + float r_ab_norm_sum_12 = r_ab_norm_12 + r_ab2_norm_12; + float r_ab_norm_sum_13 = r_ab_norm_13 + r_ab2_norm_13; + float r_ab_norm_sum_14 = r_ab_norm_14 + r_ab2_norm_14; + float r_ab_norm_sum_15 = r_ab_norm_15 + r_ab2_norm_15; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + float two_b_sq_2 = r_ab_norm_sum_2 * r_ab_norm_sum_2 - sb_sq[j + 2]; + float two_b_sq_3 = r_ab_norm_sum_3 * r_ab_norm_sum_3 - sb_sq[j + 3]; + float two_b_sq_4 = r_ab_norm_sum_4 * r_ab_norm_sum_4 - sb_sq[j + 4]; + float two_b_sq_5 = r_ab_norm_sum_5 * r_ab_norm_sum_5 - sb_sq[j + 5]; + float two_b_sq_6 = r_ab_norm_sum_6 * r_ab_norm_sum_6 - sb_sq[j + 6]; + float two_b_sq_7 = r_ab_norm_sum_7 * r_ab_norm_sum_7 - sb_sq[j + 7]; + float two_b_sq_8 = r_ab_norm_sum_8 * r_ab_norm_sum_8 - sb_sq[j + 8]; + float two_b_sq_9 = r_ab_norm_sum_9 * r_ab_norm_sum_9 - sb_sq[j + 9]; + float two_b_sq_10 = r_ab_norm_sum_10 * r_ab_norm_sum_10 - sb_sq[j + 10]; + float two_b_sq_11 = r_ab_norm_sum_11 * r_ab_norm_sum_11 - sb_sq[j + 11]; + float two_b_sq_12 = r_ab_norm_sum_12 * r_ab_norm_sum_12 - sb_sq[j + 12]; + float two_b_sq_13 = r_ab_norm_sum_13 * r_ab_norm_sum_13 - sb_sq[j + 13]; + float two_b_sq_14 = r_ab_norm_sum_14 * r_ab_norm_sum_14 - sb_sq[j + 14]; + float two_b_sq_15 = r_ab_norm_sum_15 * r_ab_norm_sum_15 - sb_sq[j + 15]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + two_b_sq_2 = two_b_sq_2 > EPS_SQ ? two_b_sq_2 : EPS_SQ; + two_b_sq_3 = two_b_sq_3 > EPS_SQ ? two_b_sq_3 : EPS_SQ; + two_b_sq_4 = two_b_sq_4 > EPS_SQ ? two_b_sq_4 : EPS_SQ; + two_b_sq_5 = two_b_sq_5 > EPS_SQ ? two_b_sq_5 : EPS_SQ; + two_b_sq_6 = two_b_sq_6 > EPS_SQ ? two_b_sq_6 : EPS_SQ; + two_b_sq_7 = two_b_sq_7 > EPS_SQ ? two_b_sq_7 : EPS_SQ; + two_b_sq_8 = two_b_sq_8 > EPS_SQ ? two_b_sq_8 : EPS_SQ; + two_b_sq_9 = two_b_sq_9 > EPS_SQ ? two_b_sq_9 : EPS_SQ; + two_b_sq_10 = two_b_sq_10 > EPS_SQ ? two_b_sq_10 : EPS_SQ; + two_b_sq_11 = two_b_sq_11 > EPS_SQ ? two_b_sq_11 : EPS_SQ; + two_b_sq_12 = two_b_sq_12 > EPS_SQ ? two_b_sq_12 : EPS_SQ; + two_b_sq_13 = two_b_sq_13 > EPS_SQ ? two_b_sq_13 : EPS_SQ; + two_b_sq_14 = two_b_sq_14 > EPS_SQ ? two_b_sq_14 : EPS_SQ; + two_b_sq_15 = two_b_sq_15 > EPS_SQ ? two_b_sq_15 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + float two_b_2 = sqrt(two_b_sq_2); + float two_b_3 = sqrt(two_b_sq_3); + float two_b_4 = sqrt(two_b_sq_4); + float two_b_5 = sqrt(two_b_sq_5); + float two_b_6 = sqrt(two_b_sq_6); + float two_b_7 = sqrt(two_b_sq_7); + float two_b_8 = sqrt(two_b_sq_8); + float two_b_9 = sqrt(two_b_sq_9); + float two_b_10 = sqrt(two_b_sq_10); + float two_b_11 = sqrt(two_b_sq_11); + float two_b_12 = sqrt(two_b_sq_12); + float two_b_13 = sqrt(two_b_sq_13); + float two_b_14 = sqrt(two_b_sq_14); + float two_b_15 = sqrt(two_b_sq_15); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_2 = (r_ab_norm_sum_2 / (two_b_2 * r_ab_norm_2 * r_ab2_norm_2)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_2 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_3 = (r_ab_norm_sum_3 / (two_b_3 * r_ab_norm_3 * r_ab2_norm_3)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_3 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_4 = (r_ab_norm_sum_4 / (two_b_4 * r_ab_norm_4 * r_ab2_norm_4)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_4 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_5 = (r_ab_norm_sum_5 / (two_b_5 * r_ab_norm_5 * r_ab2_norm_5)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_5 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_6 = (r_ab_norm_sum_6 / (two_b_6 * r_ab_norm_6 * r_ab2_norm_6)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_6 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_7 = (r_ab_norm_sum_7 / (two_b_7 * r_ab_norm_7 * r_ab2_norm_7)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_7 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_8 = (r_ab_norm_sum_8 / (two_b_8 * r_ab_norm_8 * r_ab2_norm_8)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_8 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_9 = (r_ab_norm_sum_9 / (two_b_9 * r_ab_norm_9 * r_ab2_norm_9)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_9 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_10 = (r_ab_norm_sum_10 / (two_b_10 * r_ab_norm_10 * r_ab2_norm_10)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_10 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_11 = (r_ab_norm_sum_11 / (two_b_11 * r_ab_norm_11 * r_ab2_norm_11)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_11 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_12 = (r_ab_norm_sum_12 / (two_b_12 * r_ab_norm_12 * r_ab2_norm_12)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_12 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_13 = (r_ab_norm_sum_13 / (two_b_13 * r_ab_norm_13 * r_ab2_norm_13)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_13 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_14 = (r_ab_norm_sum_14 / (two_b_14 * r_ab_norm_14 * r_ab2_norm_14)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_14 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_15 = (r_ab_norm_sum_15 / (two_b_15 * r_ab_norm_15 * r_ab2_norm_15)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_15 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + float scalar_mult_r_ab_2 = scalar_mult_common_2 * r_ab2_norm_2; + float scalar_mult_r_ab_3 = scalar_mult_common_3 * r_ab2_norm_3; + float scalar_mult_r_ab_4 = scalar_mult_common_4 * r_ab2_norm_4; + float scalar_mult_r_ab_5 = scalar_mult_common_5 * r_ab2_norm_5; + float scalar_mult_r_ab_6 = scalar_mult_common_6 * r_ab2_norm_6; + float scalar_mult_r_ab_7 = scalar_mult_common_7 * r_ab2_norm_7; + float scalar_mult_r_ab_8 = scalar_mult_common_8 * r_ab2_norm_8; + float scalar_mult_r_ab_9 = scalar_mult_common_9 * r_ab2_norm_9; + float scalar_mult_r_ab_10 = scalar_mult_common_10 * r_ab2_norm_10; + float scalar_mult_r_ab_11 = scalar_mult_common_11 * r_ab2_norm_11; + float scalar_mult_r_ab_12 = scalar_mult_common_12 * r_ab2_norm_12; + float scalar_mult_r_ab_13 = scalar_mult_common_13 * r_ab2_norm_13; + float scalar_mult_r_ab_14 = scalar_mult_common_14 * r_ab2_norm_14; + float scalar_mult_r_ab_15 = scalar_mult_common_15 * r_ab2_norm_15; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + float scalar_mult_r_ab2_2 = scalar_mult_common_2 * r_ab_norm_2; + float scalar_mult_r_ab2_3 = scalar_mult_common_3 * r_ab_norm_3; + float scalar_mult_r_ab2_4 = scalar_mult_common_4 * r_ab_norm_4; + float scalar_mult_r_ab2_5 = scalar_mult_common_5 * r_ab_norm_5; + float scalar_mult_r_ab2_6 = scalar_mult_common_6 * r_ab_norm_6; + float scalar_mult_r_ab2_7 = scalar_mult_common_7 * r_ab_norm_7; + float scalar_mult_r_ab2_8 = scalar_mult_common_8 * r_ab_norm_8; + float scalar_mult_r_ab2_9 = scalar_mult_common_9 * r_ab_norm_9; + float scalar_mult_r_ab2_10 = scalar_mult_common_10 * r_ab_norm_10; + float scalar_mult_r_ab2_11 = scalar_mult_common_11 * r_ab_norm_11; + float scalar_mult_r_ab2_12 = scalar_mult_common_12 * r_ab_norm_12; + float scalar_mult_r_ab2_13 = scalar_mult_common_13 * r_ab_norm_13; + float scalar_mult_r_ab2_14 = scalar_mult_common_14 * r_ab_norm_14; + float scalar_mult_r_ab2_15 = scalar_mult_common_15 * r_ab_norm_15; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_x_2 = r_ab_x_2 * scalar_mult_r_ab_2 + r_ab2_x_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_x_3 = r_ab_x_3 * scalar_mult_r_ab_3 + r_ab2_x_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_x_4 = r_ab_x_4 * scalar_mult_r_ab_4 + r_ab2_x_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_x_5 = r_ab_x_5 * scalar_mult_r_ab_5 + r_ab2_x_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_x_6 = r_ab_x_6 * scalar_mult_r_ab_6 + r_ab2_x_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_x_7 = r_ab_x_7 * scalar_mult_r_ab_7 + r_ab2_x_7 * scalar_mult_r_ab2_7; + float f_alpha_beta_neg_x_8 = r_ab_x_8 * scalar_mult_r_ab_8 + r_ab2_x_8 * scalar_mult_r_ab2_8; + float f_alpha_beta_neg_x_9 = r_ab_x_9 * scalar_mult_r_ab_9 + r_ab2_x_9 * scalar_mult_r_ab2_9; + float f_alpha_beta_neg_x_10 = r_ab_x_10 * scalar_mult_r_ab_10 + r_ab2_x_10 * scalar_mult_r_ab2_10; + float f_alpha_beta_neg_x_11 = r_ab_x_11 * scalar_mult_r_ab_11 + r_ab2_x_11 * scalar_mult_r_ab2_11; + float f_alpha_beta_neg_x_12 = r_ab_x_12 * scalar_mult_r_ab_12 + r_ab2_x_12 * scalar_mult_r_ab2_12; + float f_alpha_beta_neg_x_13 = r_ab_x_13 * scalar_mult_r_ab_13 + r_ab2_x_13 * scalar_mult_r_ab2_13; + float f_alpha_beta_neg_x_14 = r_ab_x_14 * scalar_mult_r_ab_14 + r_ab2_x_14 * scalar_mult_r_ab2_14; + float f_alpha_beta_neg_x_15 = r_ab_x_15 * scalar_mult_r_ab_15 + r_ab2_x_15 * scalar_mult_r_ab2_15; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_y_2 = r_ab_y_2 * scalar_mult_r_ab_2 + r_ab2_y_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_y_3 = r_ab_y_3 * scalar_mult_r_ab_3 + r_ab2_y_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_y_4 = r_ab_y_4 * scalar_mult_r_ab_4 + r_ab2_y_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_y_5 = r_ab_y_5 * scalar_mult_r_ab_5 + r_ab2_y_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_y_6 = r_ab_y_6 * scalar_mult_r_ab_6 + r_ab2_y_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_y_7 = r_ab_y_7 * scalar_mult_r_ab_7 + r_ab2_y_7 * scalar_mult_r_ab2_7; + float f_alpha_beta_neg_y_8 = r_ab_y_8 * scalar_mult_r_ab_8 + r_ab2_y_8 * scalar_mult_r_ab2_8; + float f_alpha_beta_neg_y_9 = r_ab_y_9 * scalar_mult_r_ab_9 + r_ab2_y_9 * scalar_mult_r_ab2_9; + float f_alpha_beta_neg_y_10 = r_ab_y_10 * scalar_mult_r_ab_10 + r_ab2_y_10 * scalar_mult_r_ab2_10; + float f_alpha_beta_neg_y_11 = r_ab_y_11 * scalar_mult_r_ab_11 + r_ab2_y_11 * scalar_mult_r_ab2_11; + float f_alpha_beta_neg_y_12 = r_ab_y_12 * scalar_mult_r_ab_12 + r_ab2_y_12 * scalar_mult_r_ab2_12; + float f_alpha_beta_neg_y_13 = r_ab_y_13 * scalar_mult_r_ab_13 + r_ab2_y_13 * scalar_mult_r_ab2_13; + float f_alpha_beta_neg_y_14 = r_ab_y_14 * scalar_mult_r_ab_14 + r_ab2_y_14 * scalar_mult_r_ab2_14; + float f_alpha_beta_neg_y_15 = r_ab_y_15 * scalar_mult_r_ab_15 + r_ab2_y_15 * scalar_mult_r_ab2_15; + + float w_val_0, w_val_1, w_val_2, w_val_3, w_val_4, w_val_5, w_val_6, w_val_7; + float w_val_8, w_val_9, w_val_10, w_val_11, w_val_12, w_val_13, w_val_14, w_val_15; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + float f_norm_sq_2 = f_alpha_beta_neg_x_2 * f_alpha_beta_neg_x_2 + f_alpha_beta_neg_y_2 * f_alpha_beta_neg_y_2; + float f_norm_sq_3 = f_alpha_beta_neg_x_3 * f_alpha_beta_neg_x_3 + f_alpha_beta_neg_y_3 * f_alpha_beta_neg_y_3; + float f_norm_sq_4 = f_alpha_beta_neg_x_4 * f_alpha_beta_neg_x_4 + f_alpha_beta_neg_y_4 * f_alpha_beta_neg_y_4; + float f_norm_sq_5 = f_alpha_beta_neg_x_5 * f_alpha_beta_neg_x_5 + f_alpha_beta_neg_y_5 * f_alpha_beta_neg_y_5; + float f_norm_sq_6 = f_alpha_beta_neg_x_6 * f_alpha_beta_neg_x_6 + f_alpha_beta_neg_y_6 * f_alpha_beta_neg_y_6; + float f_norm_sq_7 = f_alpha_beta_neg_x_7 * f_alpha_beta_neg_x_7 + f_alpha_beta_neg_y_7 * f_alpha_beta_neg_y_7; + float f_norm_sq_8 = f_alpha_beta_neg_x_8 * f_alpha_beta_neg_x_8 + f_alpha_beta_neg_y_8 * f_alpha_beta_neg_y_8; + float f_norm_sq_9 = f_alpha_beta_neg_x_9 * f_alpha_beta_neg_x_9 + f_alpha_beta_neg_y_9 * f_alpha_beta_neg_y_9; + float f_norm_sq_10 = f_alpha_beta_neg_x_10 * f_alpha_beta_neg_x_10 + f_alpha_beta_neg_y_10 * f_alpha_beta_neg_y_10; + float f_norm_sq_11 = f_alpha_beta_neg_x_11 * f_alpha_beta_neg_x_11 + f_alpha_beta_neg_y_11 * f_alpha_beta_neg_y_11; + float f_norm_sq_12 = f_alpha_beta_neg_x_12 * f_alpha_beta_neg_x_12 + f_alpha_beta_neg_y_12 * f_alpha_beta_neg_y_12; + float f_norm_sq_13 = f_alpha_beta_neg_x_13 * f_alpha_beta_neg_x_13 + f_alpha_beta_neg_y_13 * f_alpha_beta_neg_y_13; + float f_norm_sq_14 = f_alpha_beta_neg_x_14 * f_alpha_beta_neg_x_14 + f_alpha_beta_neg_y_14 * f_alpha_beta_neg_y_14; + float f_norm_sq_15 = f_alpha_beta_neg_x_15 * f_alpha_beta_neg_x_15 + f_alpha_beta_neg_y_15 * f_alpha_beta_neg_y_15; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + float rhs_2 = f_norm_sq_2 * cos_phi_sq; + float rhs_3 = f_norm_sq_3 * cos_phi_sq; + float rhs_4 = f_norm_sq_4 * cos_phi_sq; + float rhs_5 = f_norm_sq_5 * cos_phi_sq; + float rhs_6 = f_norm_sq_6 * cos_phi_sq; + float rhs_7 = f_norm_sq_7 * cos_phi_sq; + float rhs_8 = f_norm_sq_8 * cos_phi_sq; + float rhs_9 = f_norm_sq_9 * cos_phi_sq; + float rhs_10 = f_norm_sq_10 * cos_phi_sq; + float rhs_11 = f_norm_sq_11 * cos_phi_sq; + float rhs_12 = f_norm_sq_12 * cos_phi_sq; + float rhs_13 = f_norm_sq_13 * cos_phi_sq; + float rhs_14 = f_norm_sq_14 * cos_phi_sq; + float rhs_15 = f_norm_sq_15 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + float e_dot_f_2 = e_x_i * f_alpha_beta_neg_x_2 + e_y_i * f_alpha_beta_neg_y_2; + float e_dot_f_3 = e_x_i * f_alpha_beta_neg_x_3 + e_y_i * f_alpha_beta_neg_y_3; + float e_dot_f_4 = e_x_i * f_alpha_beta_neg_x_4 + e_y_i * f_alpha_beta_neg_y_4; + float e_dot_f_5 = e_x_i * f_alpha_beta_neg_x_5 + e_y_i * f_alpha_beta_neg_y_5; + float e_dot_f_6 = e_x_i * f_alpha_beta_neg_x_6 + e_y_i * f_alpha_beta_neg_y_6; + float e_dot_f_7 = e_x_i * f_alpha_beta_neg_x_7 + e_y_i * f_alpha_beta_neg_y_7; + float e_dot_f_8 = e_x_i * f_alpha_beta_neg_x_8 + e_y_i * f_alpha_beta_neg_y_8; + float e_dot_f_9 = e_x_i * f_alpha_beta_neg_x_9 + e_y_i * f_alpha_beta_neg_y_9; + float e_dot_f_10 = e_x_i * f_alpha_beta_neg_x_10 + e_y_i * f_alpha_beta_neg_y_10; + float e_dot_f_11 = e_x_i * f_alpha_beta_neg_x_11 + e_y_i * f_alpha_beta_neg_y_11; + float e_dot_f_12 = e_x_i * f_alpha_beta_neg_x_12 + e_y_i * f_alpha_beta_neg_y_12; + float e_dot_f_13 = e_x_i * f_alpha_beta_neg_x_13 + e_y_i * f_alpha_beta_neg_y_13; + float e_dot_f_14 = e_x_i * f_alpha_beta_neg_x_14 + e_y_i * f_alpha_beta_neg_y_14; + float e_dot_f_15 = e_x_i * f_alpha_beta_neg_x_15 + e_y_i * f_alpha_beta_neg_y_15; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + float e_dot_f_sq_2 = e_dot_f_2 * e_dot_f_2; + float e_dot_f_sq_3 = e_dot_f_3 * e_dot_f_3; + float e_dot_f_sq_4 = e_dot_f_4 * e_dot_f_4; + float e_dot_f_sq_5 = e_dot_f_5 * e_dot_f_5; + float e_dot_f_sq_6 = e_dot_f_6 * e_dot_f_6; + float e_dot_f_sq_7 = e_dot_f_7 * e_dot_f_7; + float e_dot_f_sq_8 = e_dot_f_8 * e_dot_f_8; + float e_dot_f_sq_9 = e_dot_f_9 * e_dot_f_9; + float e_dot_f_sq_10 = e_dot_f_10 * e_dot_f_10; + float e_dot_f_sq_11 = e_dot_f_11 * e_dot_f_11; + float e_dot_f_sq_12 = e_dot_f_12 * e_dot_f_12; + float e_dot_f_sq_13 = e_dot_f_13 * e_dot_f_13; + float e_dot_f_sq_14 = e_dot_f_14 * e_dot_f_14; + float e_dot_f_sq_15 = e_dot_f_15 * e_dot_f_15; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + if (e_dot_f_2 >= 0.0 || e_dot_f_sq_2 <= rhs_2) + w_val_2 = dt_neg; + else + w_val_2 = c_neg_times_dt; + if (e_dot_f_3 >= 0.0 || e_dot_f_sq_3 <= rhs_3) + w_val_3 = dt_neg; + else + w_val_3 = c_neg_times_dt; + if (e_dot_f_4 >= 0.0 || e_dot_f_sq_4 <= rhs_4) + w_val_4 = dt_neg; + else + w_val_4 = c_neg_times_dt; + if (e_dot_f_5 >= 0.0 || e_dot_f_sq_5 <= rhs_5) + w_val_5 = dt_neg; + else + w_val_5 = c_neg_times_dt; + if (e_dot_f_6 >= 0.0 || e_dot_f_sq_6 <= rhs_6) + w_val_6 = dt_neg; + else + w_val_6 = c_neg_times_dt; + if (e_dot_f_7 >= 0.0 || e_dot_f_sq_7 <= rhs_7) + w_val_7 = dt_neg; + else + w_val_7 = c_neg_times_dt; + if (e_dot_f_8 >= 0.0 || e_dot_f_sq_8 <= rhs_8) + w_val_8 = dt_neg; + else + w_val_8 = c_neg_times_dt; + if (e_dot_f_9 >= 0.0 || e_dot_f_sq_9 <= rhs_9) + w_val_9 = dt_neg; + else + w_val_9 = c_neg_times_dt; + if (e_dot_f_10 >= 0.0 || e_dot_f_sq_10 <= rhs_10) + w_val_10 = dt_neg; + else + w_val_10 = c_neg_times_dt; + if (e_dot_f_11 >= 0.0 || e_dot_f_sq_11 <= rhs_11) + w_val_11 = dt_neg; + else + w_val_11 = c_neg_times_dt; + if (e_dot_f_12 >= 0.0 || e_dot_f_sq_12 <= rhs_12) + w_val_12 = dt_neg; + else + w_val_12 = c_neg_times_dt; + if (e_dot_f_13 >= 0.0 || e_dot_f_sq_13 <= rhs_13) + w_val_13 = dt_neg; + else + w_val_13 = c_neg_times_dt; + if (e_dot_f_14 >= 0.0 || e_dot_f_sq_14 <= rhs_14) + w_val_14 = dt_neg; + else + w_val_14 = c_neg_times_dt; + if (e_dot_f_15 >= 0.0 || e_dot_f_sq_15 <= rhs_15) + w_val_15 = dt_neg; + else + w_val_15 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_x_2 += f_alpha_beta_neg_x_2 * w_val_2; + acc_x_3 += f_alpha_beta_neg_x_3 * w_val_3; + acc_x_4 += f_alpha_beta_neg_x_4 * w_val_4; + acc_x_5 += f_alpha_beta_neg_x_5 * w_val_5; + acc_x_6 += f_alpha_beta_neg_x_6 * w_val_6; + acc_x_7 += f_alpha_beta_neg_x_7 * w_val_7; + acc_x_8 += f_alpha_beta_neg_x_8 * w_val_8; + acc_x_9 += f_alpha_beta_neg_x_9 * w_val_9; + acc_x_10 += f_alpha_beta_neg_x_10 * w_val_10; + acc_x_11 += f_alpha_beta_neg_x_11 * w_val_11; + acc_x_12 += f_alpha_beta_neg_x_12 * w_val_12; + acc_x_13 += f_alpha_beta_neg_x_13 * w_val_13; + acc_x_14 += f_alpha_beta_neg_x_14 * w_val_14; + acc_x_15 += f_alpha_beta_neg_x_15 * w_val_15; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + acc_y_2 += f_alpha_beta_neg_y_2 * w_val_2; + acc_y_3 += f_alpha_beta_neg_y_3 * w_val_3; + acc_y_4 += f_alpha_beta_neg_y_4 * w_val_4; + acc_y_5 += f_alpha_beta_neg_y_5 * w_val_5; + acc_y_6 += f_alpha_beta_neg_y_6 * w_val_6; + acc_y_7 += f_alpha_beta_neg_y_7 * w_val_7; + acc_y_8 += f_alpha_beta_neg_y_8 * w_val_8; + acc_y_9 += f_alpha_beta_neg_y_9 * w_val_9; + acc_y_10 += f_alpha_beta_neg_y_10 * w_val_10; + acc_y_11 += f_alpha_beta_neg_y_11 * w_val_11; + acc_y_12 += f_alpha_beta_neg_y_12 * w_val_12; + acc_y_13 += f_alpha_beta_neg_y_13 * w_val_13; + acc_y_14 += f_alpha_beta_neg_y_14 * w_val_14; + acc_y_15 += f_alpha_beta_neg_y_15 * w_val_15; + } + + for (; j < i; j++) { + float r_ab_x = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_y = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_norm = r_ab_norm_lut[j * n_p + i]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1 + acc_x_2 + acc_x_3 + acc_x_4 + acc_x_5 + acc_x_6 + acc_x_7 + acc_x_8 + acc_x_9 + acc_x_10 + acc_x_11 + acc_x_12 + acc_x_13 + acc_x_14 + acc_x_15; + f_alpha_dt_y += acc_y_0 + acc_y_1 + acc_y_2 + acc_y_3 + acc_y_4 + acc_y_5 + acc_y_6 + acc_y_7 + acc_y_8 + acc_y_9 + acc_y_10 + acc_y_11 + acc_y_12 + acc_y_13 + acc_y_14 + acc_y_15; + + acc_x_0 = 0.0f; + acc_x_1 = 0.0f; + acc_x_2 = 0.0f; + acc_x_3 = 0.0f; + acc_x_4 = 0.0f; + acc_x_5 = 0.0f; + acc_x_6 = 0.0f; + acc_x_7 = 0.0f; + acc_x_8 = 0.0f; + acc_x_9 = 0.0f; + acc_x_10 = 0.0f; + acc_x_11 = 0.0f; + acc_x_12 = 0.0f; + acc_x_13 = 0.0f; + acc_x_14 = 0.0f; + acc_x_15 = 0.0f; + acc_y_0 = 0.0f; + acc_y_1 = 0.0f; + acc_y_2 = 0.0f; + acc_y_3 = 0.0f; + acc_y_4 = 0.0f; + acc_y_5 = 0.0f; + acc_y_6 = 0.0f; + acc_y_7 = 0.0f; + acc_y_8 = 0.0f; + acc_y_9 = 0.0f; + acc_y_10 = 0.0f; + acc_y_11 = 0.0f; + acc_y_12 = 0.0f; + acc_y_13 = 0.0f; + acc_y_14 = 0.0f; + acc_y_15 = 0.0f; + + for (j = i + 1; j + 16 < n_p; j += 16) { + float r_ab_x_0 = r_ab_x_lut[i * n_p + j]; + float r_ab_x_1 = r_ab_x_lut[i * n_p + j + 1]; + float r_ab_x_2 = r_ab_x_lut[i * n_p + j + 2]; + float r_ab_x_3 = r_ab_x_lut[i * n_p + j + 3]; + float r_ab_x_4 = r_ab_x_lut[i * n_p + j + 4]; + float r_ab_x_5 = r_ab_x_lut[i * n_p + j + 5]; + float r_ab_x_6 = r_ab_x_lut[i * n_p + j + 6]; + float r_ab_x_7 = r_ab_x_lut[i * n_p + j + 7]; + float r_ab_x_8 = r_ab_x_lut[i * n_p + j + 8]; + float r_ab_x_9 = r_ab_x_lut[i * n_p + j + 9]; + float r_ab_x_10 = r_ab_x_lut[i * n_p + j + 10]; + float r_ab_x_11 = r_ab_x_lut[i * n_p + j + 11]; + float r_ab_x_12 = r_ab_x_lut[i * n_p + j + 12]; + float r_ab_x_13 = r_ab_x_lut[i * n_p + j + 13]; + float r_ab_x_14 = r_ab_x_lut[i * n_p + j + 14]; + float r_ab_x_15 = r_ab_x_lut[i * n_p + j + 15]; + + float r_ab_y_0 = r_ab_y_lut[i * n_p + j]; + float r_ab_y_1 = r_ab_y_lut[i * n_p + j + 1]; + float r_ab_y_2 = r_ab_y_lut[i * n_p + j + 2]; + float r_ab_y_3 = r_ab_y_lut[i * n_p + j + 3]; + float r_ab_y_4 = r_ab_y_lut[i * n_p + j + 4]; + float r_ab_y_5 = r_ab_y_lut[i * n_p + j + 5]; + float r_ab_y_6 = r_ab_y_lut[i * n_p + j + 6]; + float r_ab_y_7 = r_ab_y_lut[i * n_p + j + 7]; + float r_ab_y_8 = r_ab_y_lut[i * n_p + j + 8]; + float r_ab_y_9 = r_ab_y_lut[i * n_p + j + 9]; + float r_ab_y_10 = r_ab_y_lut[i * n_p + j + 10]; + float r_ab_y_11 = r_ab_y_lut[i * n_p + j + 11]; + float r_ab_y_12 = r_ab_y_lut[i * n_p + j + 12]; + float r_ab_y_13 = r_ab_y_lut[i * n_p + j + 13]; + float r_ab_y_14 = r_ab_y_lut[i * n_p + j + 14]; + float r_ab_y_15 = r_ab_y_lut[i * n_p + j + 15]; + + float r_ab_norm_0 = r_ab_norm_lut[i * n_p + j]; + float r_ab_norm_1 = r_ab_norm_lut[i * n_p + j + 1]; + float r_ab_norm_2 = r_ab_norm_lut[i * n_p + j + 2]; + float r_ab_norm_3 = r_ab_norm_lut[i * n_p + j + 3]; + float r_ab_norm_4 = r_ab_norm_lut[i * n_p + j + 4]; + float r_ab_norm_5 = r_ab_norm_lut[i * n_p + j + 5]; + float r_ab_norm_6 = r_ab_norm_lut[i * n_p + j + 6]; + float r_ab_norm_7 = r_ab_norm_lut[i * n_p + j + 7]; + float r_ab_norm_8 = r_ab_norm_lut[i * n_p + j + 8]; + float r_ab_norm_9 = r_ab_norm_lut[i * n_p + j + 9]; + float r_ab_norm_10 = r_ab_norm_lut[i * n_p + j + 10]; + float r_ab_norm_11 = r_ab_norm_lut[i * n_p + j + 11]; + float r_ab_norm_12 = r_ab_norm_lut[i * n_p + j + 12]; + float r_ab_norm_13 = r_ab_norm_lut[i * n_p + j + 13]; + float r_ab_norm_14 = r_ab_norm_lut[i * n_p + j + 14]; + float r_ab_norm_15 = r_ab_norm_lut[i * n_p + j + 15]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + r_ab_norm_2 = r_ab_norm_2 > EPS ? r_ab_norm_2 : EPS; + r_ab_norm_3 = r_ab_norm_3 > EPS ? r_ab_norm_3 : EPS; + r_ab_norm_4 = r_ab_norm_4 > EPS ? r_ab_norm_4 : EPS; + r_ab_norm_5 = r_ab_norm_5 > EPS ? r_ab_norm_5 : EPS; + r_ab_norm_6 = r_ab_norm_6 > EPS ? r_ab_norm_6 : EPS; + r_ab_norm_7 = r_ab_norm_7 > EPS ? r_ab_norm_7 : EPS; + r_ab_norm_8 = r_ab_norm_8 > EPS ? r_ab_norm_8 : EPS; + r_ab_norm_9 = r_ab_norm_9 > EPS ? r_ab_norm_9 : EPS; + r_ab_norm_10 = r_ab_norm_10 > EPS ? r_ab_norm_10 : EPS; + r_ab_norm_11 = r_ab_norm_11 > EPS ? r_ab_norm_11 : EPS; + r_ab_norm_12 = r_ab_norm_12 > EPS ? r_ab_norm_12 : EPS; + r_ab_norm_13 = r_ab_norm_13 > EPS ? r_ab_norm_13 : EPS; + r_ab_norm_14 = r_ab_norm_14 > EPS ? r_ab_norm_14 : EPS; + r_ab_norm_15 = r_ab_norm_15 > EPS ? r_ab_norm_15 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + float r_ab2_x_2 = r_ab_x_2 - sb_e_x[j + 2]; + float r_ab2_x_3 = r_ab_x_3 - sb_e_x[j + 3]; + float r_ab2_x_4 = r_ab_x_4 - sb_e_x[j + 4]; + float r_ab2_x_5 = r_ab_x_5 - sb_e_x[j + 5]; + float r_ab2_x_6 = r_ab_x_6 - sb_e_x[j + 6]; + float r_ab2_x_7 = r_ab_x_7 - sb_e_x[j + 7]; + float r_ab2_x_8 = r_ab_x_8 - sb_e_x[j + 8]; + float r_ab2_x_9 = r_ab_x_9 - sb_e_x[j + 9]; + float r_ab2_x_10 = r_ab_x_10 - sb_e_x[j + 10]; + float r_ab2_x_11 = r_ab_x_11 - sb_e_x[j + 11]; + float r_ab2_x_12 = r_ab_x_12 - sb_e_x[j + 12]; + float r_ab2_x_13 = r_ab_x_13 - sb_e_x[j + 13]; + float r_ab2_x_14 = r_ab_x_14 - sb_e_x[j + 14]; + float r_ab2_x_15 = r_ab_x_15 - sb_e_x[j + 15]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + float r_ab2_y_2 = r_ab_y_2 - sb_e_y[j + 2]; + float r_ab2_y_3 = r_ab_y_3 - sb_e_y[j + 3]; + float r_ab2_y_4 = r_ab_y_4 - sb_e_y[j + 4]; + float r_ab2_y_5 = r_ab_y_5 - sb_e_y[j + 5]; + float r_ab2_y_6 = r_ab_y_6 - sb_e_y[j + 6]; + float r_ab2_y_7 = r_ab_y_7 - sb_e_y[j + 7]; + float r_ab2_y_8 = r_ab_y_8 - sb_e_y[j + 8]; + float r_ab2_y_9 = r_ab_y_9 - sb_e_y[j + 9]; + float r_ab2_y_10 = r_ab_y_10 - sb_e_y[j + 10]; + float r_ab2_y_11 = r_ab_y_11 - sb_e_y[j + 11]; + float r_ab2_y_12 = r_ab_y_12 - sb_e_y[j + 12]; + float r_ab2_y_13 = r_ab_y_13 - sb_e_y[j + 13]; + float r_ab2_y_14 = r_ab_y_14 - sb_e_y[j + 14]; + float r_ab2_y_15 = r_ab_y_15 - sb_e_y[j + 15]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + float r_ab2_norm_2 = sqrt(r_ab2_x_2 * r_ab2_x_2 + r_ab2_y_2 * r_ab2_y_2); + float r_ab2_norm_3 = sqrt(r_ab2_x_3 * r_ab2_x_3 + r_ab2_y_3 * r_ab2_y_3); + float r_ab2_norm_4 = sqrt(r_ab2_x_4 * r_ab2_x_4 + r_ab2_y_4 * r_ab2_y_4); + float r_ab2_norm_5 = sqrt(r_ab2_x_5 * r_ab2_x_5 + r_ab2_y_5 * r_ab2_y_5); + float r_ab2_norm_6 = sqrt(r_ab2_x_6 * r_ab2_x_6 + r_ab2_y_6 * r_ab2_y_6); + float r_ab2_norm_7 = sqrt(r_ab2_x_7 * r_ab2_x_7 + r_ab2_y_7 * r_ab2_y_7); + float r_ab2_norm_8 = sqrt(r_ab2_x_8 * r_ab2_x_8 + r_ab2_y_8 * r_ab2_y_8); + float r_ab2_norm_9 = sqrt(r_ab2_x_9 * r_ab2_x_9 + r_ab2_y_9 * r_ab2_y_9); + float r_ab2_norm_10 = sqrt(r_ab2_x_10 * r_ab2_x_10 + r_ab2_y_10 * r_ab2_y_10); + float r_ab2_norm_11 = sqrt(r_ab2_x_11 * r_ab2_x_11 + r_ab2_y_11 * r_ab2_y_11); + float r_ab2_norm_12 = sqrt(r_ab2_x_12 * r_ab2_x_12 + r_ab2_y_12 * r_ab2_y_12); + float r_ab2_norm_13 = sqrt(r_ab2_x_13 * r_ab2_x_13 + r_ab2_y_13 * r_ab2_y_13); + float r_ab2_norm_14 = sqrt(r_ab2_x_14 * r_ab2_x_14 + r_ab2_y_14 * r_ab2_y_14); + float r_ab2_norm_15 = sqrt(r_ab2_x_15 * r_ab2_x_15 + r_ab2_y_15 * r_ab2_y_15); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + r_ab2_norm_2 = r_ab2_norm_2 > EPS ? r_ab2_norm_2 : EPS; + r_ab2_norm_3 = r_ab2_norm_3 > EPS ? r_ab2_norm_3 : EPS; + r_ab2_norm_4 = r_ab2_norm_4 > EPS ? r_ab2_norm_4 : EPS; + r_ab2_norm_5 = r_ab2_norm_5 > EPS ? r_ab2_norm_5 : EPS; + r_ab2_norm_6 = r_ab2_norm_6 > EPS ? r_ab2_norm_6 : EPS; + r_ab2_norm_7 = r_ab2_norm_7 > EPS ? r_ab2_norm_7 : EPS; + r_ab2_norm_8 = r_ab2_norm_8 > EPS ? r_ab2_norm_8 : EPS; + r_ab2_norm_9 = r_ab2_norm_9 > EPS ? r_ab2_norm_9 : EPS; + r_ab2_norm_10 = r_ab2_norm_10 > EPS ? r_ab2_norm_10 : EPS; + r_ab2_norm_11 = r_ab2_norm_11 > EPS ? r_ab2_norm_11 : EPS; + r_ab2_norm_12 = r_ab2_norm_12 > EPS ? r_ab2_norm_12 : EPS; + r_ab2_norm_13 = r_ab2_norm_13 > EPS ? r_ab2_norm_13 : EPS; + r_ab2_norm_14 = r_ab2_norm_14 > EPS ? r_ab2_norm_14 : EPS; + r_ab2_norm_15 = r_ab2_norm_15 > EPS ? r_ab2_norm_15 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + float r_ab_norm_sum_2 = r_ab_norm_2 + r_ab2_norm_2; + float r_ab_norm_sum_3 = r_ab_norm_3 + r_ab2_norm_3; + float r_ab_norm_sum_4 = r_ab_norm_4 + r_ab2_norm_4; + float r_ab_norm_sum_5 = r_ab_norm_5 + r_ab2_norm_5; + float r_ab_norm_sum_6 = r_ab_norm_6 + r_ab2_norm_6; + float r_ab_norm_sum_7 = r_ab_norm_7 + r_ab2_norm_7; + float r_ab_norm_sum_8 = r_ab_norm_8 + r_ab2_norm_8; + float r_ab_norm_sum_9 = r_ab_norm_9 + r_ab2_norm_9; + float r_ab_norm_sum_10 = r_ab_norm_10 + r_ab2_norm_10; + float r_ab_norm_sum_11 = r_ab_norm_11 + r_ab2_norm_11; + float r_ab_norm_sum_12 = r_ab_norm_12 + r_ab2_norm_12; + float r_ab_norm_sum_13 = r_ab_norm_13 + r_ab2_norm_13; + float r_ab_norm_sum_14 = r_ab_norm_14 + r_ab2_norm_14; + float r_ab_norm_sum_15 = r_ab_norm_15 + r_ab2_norm_15; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + float two_b_sq_2 = r_ab_norm_sum_2 * r_ab_norm_sum_2 - sb_sq[j + 2]; + float two_b_sq_3 = r_ab_norm_sum_3 * r_ab_norm_sum_3 - sb_sq[j + 3]; + float two_b_sq_4 = r_ab_norm_sum_4 * r_ab_norm_sum_4 - sb_sq[j + 4]; + float two_b_sq_5 = r_ab_norm_sum_5 * r_ab_norm_sum_5 - sb_sq[j + 5]; + float two_b_sq_6 = r_ab_norm_sum_6 * r_ab_norm_sum_6 - sb_sq[j + 6]; + float two_b_sq_7 = r_ab_norm_sum_7 * r_ab_norm_sum_7 - sb_sq[j + 7]; + float two_b_sq_8 = r_ab_norm_sum_8 * r_ab_norm_sum_8 - sb_sq[j + 8]; + float two_b_sq_9 = r_ab_norm_sum_9 * r_ab_norm_sum_9 - sb_sq[j + 9]; + float two_b_sq_10 = r_ab_norm_sum_10 * r_ab_norm_sum_10 - sb_sq[j + 10]; + float two_b_sq_11 = r_ab_norm_sum_11 * r_ab_norm_sum_11 - sb_sq[j + 11]; + float two_b_sq_12 = r_ab_norm_sum_12 * r_ab_norm_sum_12 - sb_sq[j + 12]; + float two_b_sq_13 = r_ab_norm_sum_13 * r_ab_norm_sum_13 - sb_sq[j + 13]; + float two_b_sq_14 = r_ab_norm_sum_14 * r_ab_norm_sum_14 - sb_sq[j + 14]; + float two_b_sq_15 = r_ab_norm_sum_15 * r_ab_norm_sum_15 - sb_sq[j + 15]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + two_b_sq_2 = two_b_sq_2 > EPS_SQ ? two_b_sq_2 : EPS_SQ; + two_b_sq_3 = two_b_sq_3 > EPS_SQ ? two_b_sq_3 : EPS_SQ; + two_b_sq_4 = two_b_sq_4 > EPS_SQ ? two_b_sq_4 : EPS_SQ; + two_b_sq_5 = two_b_sq_5 > EPS_SQ ? two_b_sq_5 : EPS_SQ; + two_b_sq_6 = two_b_sq_6 > EPS_SQ ? two_b_sq_6 : EPS_SQ; + two_b_sq_7 = two_b_sq_7 > EPS_SQ ? two_b_sq_7 : EPS_SQ; + two_b_sq_8 = two_b_sq_8 > EPS_SQ ? two_b_sq_8 : EPS_SQ; + two_b_sq_9 = two_b_sq_9 > EPS_SQ ? two_b_sq_9 : EPS_SQ; + two_b_sq_10 = two_b_sq_10 > EPS_SQ ? two_b_sq_10 : EPS_SQ; + two_b_sq_11 = two_b_sq_11 > EPS_SQ ? two_b_sq_11 : EPS_SQ; + two_b_sq_12 = two_b_sq_12 > EPS_SQ ? two_b_sq_12 : EPS_SQ; + two_b_sq_13 = two_b_sq_13 > EPS_SQ ? two_b_sq_13 : EPS_SQ; + two_b_sq_14 = two_b_sq_14 > EPS_SQ ? two_b_sq_14 : EPS_SQ; + two_b_sq_15 = two_b_sq_15 > EPS_SQ ? two_b_sq_15 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + float two_b_2 = sqrt(two_b_sq_2); + float two_b_3 = sqrt(two_b_sq_3); + float two_b_4 = sqrt(two_b_sq_4); + float two_b_5 = sqrt(two_b_sq_5); + float two_b_6 = sqrt(two_b_sq_6); + float two_b_7 = sqrt(two_b_sq_7); + float two_b_8 = sqrt(two_b_sq_8); + float two_b_9 = sqrt(two_b_sq_9); + float two_b_10 = sqrt(two_b_sq_10); + float two_b_11 = sqrt(two_b_sq_11); + float two_b_12 = sqrt(two_b_sq_12); + float two_b_13 = sqrt(two_b_sq_13); + float two_b_14 = sqrt(two_b_sq_14); + float two_b_15 = sqrt(two_b_sq_15); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_2 = (r_ab_norm_sum_2 / (two_b_2 * r_ab_norm_2 * r_ab2_norm_2)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_2 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_3 = (r_ab_norm_sum_3 / (two_b_3 * r_ab_norm_3 * r_ab2_norm_3)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_3 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_4 = (r_ab_norm_sum_4 / (two_b_4 * r_ab_norm_4 * r_ab2_norm_4)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_4 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_5 = (r_ab_norm_sum_5 / (two_b_5 * r_ab_norm_5 * r_ab2_norm_5)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_5 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_6 = (r_ab_norm_sum_6 / (two_b_6 * r_ab_norm_6 * r_ab2_norm_6)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_6 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_7 = (r_ab_norm_sum_7 / (two_b_7 * r_ab_norm_7 * r_ab2_norm_7)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_7 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_8 = (r_ab_norm_sum_8 / (two_b_8 * r_ab_norm_8 * r_ab2_norm_8)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_8 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_9 = (r_ab_norm_sum_9 / (two_b_9 * r_ab_norm_9 * r_ab2_norm_9)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_9 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_10 = (r_ab_norm_sum_10 / (two_b_10 * r_ab_norm_10 * r_ab2_norm_10)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_10 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_11 = (r_ab_norm_sum_11 / (two_b_11 * r_ab_norm_11 * r_ab2_norm_11)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_11 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_12 = (r_ab_norm_sum_12 / (two_b_12 * r_ab_norm_12 * r_ab2_norm_12)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_12 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_13 = (r_ab_norm_sum_13 / (two_b_13 * r_ab_norm_13 * r_ab2_norm_13)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_13 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_14 = (r_ab_norm_sum_14 / (two_b_14 * r_ab_norm_14 * r_ab2_norm_14)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_14 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_15 = (r_ab_norm_sum_15 / (two_b_15 * r_ab_norm_15 * r_ab2_norm_15)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_15 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + float scalar_mult_r_ab_2 = scalar_mult_common_2 * r_ab2_norm_2; + float scalar_mult_r_ab_3 = scalar_mult_common_3 * r_ab2_norm_3; + float scalar_mult_r_ab_4 = scalar_mult_common_4 * r_ab2_norm_4; + float scalar_mult_r_ab_5 = scalar_mult_common_5 * r_ab2_norm_5; + float scalar_mult_r_ab_6 = scalar_mult_common_6 * r_ab2_norm_6; + float scalar_mult_r_ab_7 = scalar_mult_common_7 * r_ab2_norm_7; + float scalar_mult_r_ab_8 = scalar_mult_common_8 * r_ab2_norm_8; + float scalar_mult_r_ab_9 = scalar_mult_common_9 * r_ab2_norm_9; + float scalar_mult_r_ab_10 = scalar_mult_common_10 * r_ab2_norm_10; + float scalar_mult_r_ab_11 = scalar_mult_common_11 * r_ab2_norm_11; + float scalar_mult_r_ab_12 = scalar_mult_common_12 * r_ab2_norm_12; + float scalar_mult_r_ab_13 = scalar_mult_common_13 * r_ab2_norm_13; + float scalar_mult_r_ab_14 = scalar_mult_common_14 * r_ab2_norm_14; + float scalar_mult_r_ab_15 = scalar_mult_common_15 * r_ab2_norm_15; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + float scalar_mult_r_ab2_2 = scalar_mult_common_2 * r_ab_norm_2; + float scalar_mult_r_ab2_3 = scalar_mult_common_3 * r_ab_norm_3; + float scalar_mult_r_ab2_4 = scalar_mult_common_4 * r_ab_norm_4; + float scalar_mult_r_ab2_5 = scalar_mult_common_5 * r_ab_norm_5; + float scalar_mult_r_ab2_6 = scalar_mult_common_6 * r_ab_norm_6; + float scalar_mult_r_ab2_7 = scalar_mult_common_7 * r_ab_norm_7; + float scalar_mult_r_ab2_8 = scalar_mult_common_8 * r_ab_norm_8; + float scalar_mult_r_ab2_9 = scalar_mult_common_9 * r_ab_norm_9; + float scalar_mult_r_ab2_10 = scalar_mult_common_10 * r_ab_norm_10; + float scalar_mult_r_ab2_11 = scalar_mult_common_11 * r_ab_norm_11; + float scalar_mult_r_ab2_12 = scalar_mult_common_12 * r_ab_norm_12; + float scalar_mult_r_ab2_13 = scalar_mult_common_13 * r_ab_norm_13; + float scalar_mult_r_ab2_14 = scalar_mult_common_14 * r_ab_norm_14; + float scalar_mult_r_ab2_15 = scalar_mult_common_15 * r_ab_norm_15; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_x_2 = r_ab_x_2 * scalar_mult_r_ab_2 + r_ab2_x_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_x_3 = r_ab_x_3 * scalar_mult_r_ab_3 + r_ab2_x_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_x_4 = r_ab_x_4 * scalar_mult_r_ab_4 + r_ab2_x_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_x_5 = r_ab_x_5 * scalar_mult_r_ab_5 + r_ab2_x_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_x_6 = r_ab_x_6 * scalar_mult_r_ab_6 + r_ab2_x_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_x_7 = r_ab_x_7 * scalar_mult_r_ab_7 + r_ab2_x_7 * scalar_mult_r_ab2_7; + float f_alpha_beta_neg_x_8 = r_ab_x_8 * scalar_mult_r_ab_8 + r_ab2_x_8 * scalar_mult_r_ab2_8; + float f_alpha_beta_neg_x_9 = r_ab_x_9 * scalar_mult_r_ab_9 + r_ab2_x_9 * scalar_mult_r_ab2_9; + float f_alpha_beta_neg_x_10 = r_ab_x_10 * scalar_mult_r_ab_10 + r_ab2_x_10 * scalar_mult_r_ab2_10; + float f_alpha_beta_neg_x_11 = r_ab_x_11 * scalar_mult_r_ab_11 + r_ab2_x_11 * scalar_mult_r_ab2_11; + float f_alpha_beta_neg_x_12 = r_ab_x_12 * scalar_mult_r_ab_12 + r_ab2_x_12 * scalar_mult_r_ab2_12; + float f_alpha_beta_neg_x_13 = r_ab_x_13 * scalar_mult_r_ab_13 + r_ab2_x_13 * scalar_mult_r_ab2_13; + float f_alpha_beta_neg_x_14 = r_ab_x_14 * scalar_mult_r_ab_14 + r_ab2_x_14 * scalar_mult_r_ab2_14; + float f_alpha_beta_neg_x_15 = r_ab_x_15 * scalar_mult_r_ab_15 + r_ab2_x_15 * scalar_mult_r_ab2_15; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_y_2 = r_ab_y_2 * scalar_mult_r_ab_2 + r_ab2_y_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_y_3 = r_ab_y_3 * scalar_mult_r_ab_3 + r_ab2_y_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_y_4 = r_ab_y_4 * scalar_mult_r_ab_4 + r_ab2_y_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_y_5 = r_ab_y_5 * scalar_mult_r_ab_5 + r_ab2_y_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_y_6 = r_ab_y_6 * scalar_mult_r_ab_6 + r_ab2_y_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_y_7 = r_ab_y_7 * scalar_mult_r_ab_7 + r_ab2_y_7 * scalar_mult_r_ab2_7; + float f_alpha_beta_neg_y_8 = r_ab_y_8 * scalar_mult_r_ab_8 + r_ab2_y_8 * scalar_mult_r_ab2_8; + float f_alpha_beta_neg_y_9 = r_ab_y_9 * scalar_mult_r_ab_9 + r_ab2_y_9 * scalar_mult_r_ab2_9; + float f_alpha_beta_neg_y_10 = r_ab_y_10 * scalar_mult_r_ab_10 + r_ab2_y_10 * scalar_mult_r_ab2_10; + float f_alpha_beta_neg_y_11 = r_ab_y_11 * scalar_mult_r_ab_11 + r_ab2_y_11 * scalar_mult_r_ab2_11; + float f_alpha_beta_neg_y_12 = r_ab_y_12 * scalar_mult_r_ab_12 + r_ab2_y_12 * scalar_mult_r_ab2_12; + float f_alpha_beta_neg_y_13 = r_ab_y_13 * scalar_mult_r_ab_13 + r_ab2_y_13 * scalar_mult_r_ab2_13; + float f_alpha_beta_neg_y_14 = r_ab_y_14 * scalar_mult_r_ab_14 + r_ab2_y_14 * scalar_mult_r_ab2_14; + float f_alpha_beta_neg_y_15 = r_ab_y_15 * scalar_mult_r_ab_15 + r_ab2_y_15 * scalar_mult_r_ab2_15; + + float w_val_0, w_val_1, w_val_2, w_val_3, w_val_4, w_val_5, w_val_6, w_val_7; + float w_val_8, w_val_9, w_val_10, w_val_11, w_val_12, w_val_13, w_val_14, w_val_15; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + float f_norm_sq_2 = f_alpha_beta_neg_x_2 * f_alpha_beta_neg_x_2 + f_alpha_beta_neg_y_2 * f_alpha_beta_neg_y_2; + float f_norm_sq_3 = f_alpha_beta_neg_x_3 * f_alpha_beta_neg_x_3 + f_alpha_beta_neg_y_3 * f_alpha_beta_neg_y_3; + float f_norm_sq_4 = f_alpha_beta_neg_x_4 * f_alpha_beta_neg_x_4 + f_alpha_beta_neg_y_4 * f_alpha_beta_neg_y_4; + float f_norm_sq_5 = f_alpha_beta_neg_x_5 * f_alpha_beta_neg_x_5 + f_alpha_beta_neg_y_5 * f_alpha_beta_neg_y_5; + float f_norm_sq_6 = f_alpha_beta_neg_x_6 * f_alpha_beta_neg_x_6 + f_alpha_beta_neg_y_6 * f_alpha_beta_neg_y_6; + float f_norm_sq_7 = f_alpha_beta_neg_x_7 * f_alpha_beta_neg_x_7 + f_alpha_beta_neg_y_7 * f_alpha_beta_neg_y_7; + float f_norm_sq_8 = f_alpha_beta_neg_x_8 * f_alpha_beta_neg_x_8 + f_alpha_beta_neg_y_8 * f_alpha_beta_neg_y_8; + float f_norm_sq_9 = f_alpha_beta_neg_x_9 * f_alpha_beta_neg_x_9 + f_alpha_beta_neg_y_9 * f_alpha_beta_neg_y_9; + float f_norm_sq_10 = f_alpha_beta_neg_x_10 * f_alpha_beta_neg_x_10 + f_alpha_beta_neg_y_10 * f_alpha_beta_neg_y_10; + float f_norm_sq_11 = f_alpha_beta_neg_x_11 * f_alpha_beta_neg_x_11 + f_alpha_beta_neg_y_11 * f_alpha_beta_neg_y_11; + float f_norm_sq_12 = f_alpha_beta_neg_x_12 * f_alpha_beta_neg_x_12 + f_alpha_beta_neg_y_12 * f_alpha_beta_neg_y_12; + float f_norm_sq_13 = f_alpha_beta_neg_x_13 * f_alpha_beta_neg_x_13 + f_alpha_beta_neg_y_13 * f_alpha_beta_neg_y_13; + float f_norm_sq_14 = f_alpha_beta_neg_x_14 * f_alpha_beta_neg_x_14 + f_alpha_beta_neg_y_14 * f_alpha_beta_neg_y_14; + float f_norm_sq_15 = f_alpha_beta_neg_x_15 * f_alpha_beta_neg_x_15 + f_alpha_beta_neg_y_15 * f_alpha_beta_neg_y_15; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + float rhs_2 = f_norm_sq_2 * cos_phi_sq; + float rhs_3 = f_norm_sq_3 * cos_phi_sq; + float rhs_4 = f_norm_sq_4 * cos_phi_sq; + float rhs_5 = f_norm_sq_5 * cos_phi_sq; + float rhs_6 = f_norm_sq_6 * cos_phi_sq; + float rhs_7 = f_norm_sq_7 * cos_phi_sq; + float rhs_8 = f_norm_sq_8 * cos_phi_sq; + float rhs_9 = f_norm_sq_9 * cos_phi_sq; + float rhs_10 = f_norm_sq_10 * cos_phi_sq; + float rhs_11 = f_norm_sq_11 * cos_phi_sq; + float rhs_12 = f_norm_sq_12 * cos_phi_sq; + float rhs_13 = f_norm_sq_13 * cos_phi_sq; + float rhs_14 = f_norm_sq_14 * cos_phi_sq; + float rhs_15 = f_norm_sq_15 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + float e_dot_f_2 = e_x_i * f_alpha_beta_neg_x_2 + e_y_i * f_alpha_beta_neg_y_2; + float e_dot_f_3 = e_x_i * f_alpha_beta_neg_x_3 + e_y_i * f_alpha_beta_neg_y_3; + float e_dot_f_4 = e_x_i * f_alpha_beta_neg_x_4 + e_y_i * f_alpha_beta_neg_y_4; + float e_dot_f_5 = e_x_i * f_alpha_beta_neg_x_5 + e_y_i * f_alpha_beta_neg_y_5; + float e_dot_f_6 = e_x_i * f_alpha_beta_neg_x_6 + e_y_i * f_alpha_beta_neg_y_6; + float e_dot_f_7 = e_x_i * f_alpha_beta_neg_x_7 + e_y_i * f_alpha_beta_neg_y_7; + float e_dot_f_8 = e_x_i * f_alpha_beta_neg_x_8 + e_y_i * f_alpha_beta_neg_y_8; + float e_dot_f_9 = e_x_i * f_alpha_beta_neg_x_9 + e_y_i * f_alpha_beta_neg_y_9; + float e_dot_f_10 = e_x_i * f_alpha_beta_neg_x_10 + e_y_i * f_alpha_beta_neg_y_10; + float e_dot_f_11 = e_x_i * f_alpha_beta_neg_x_11 + e_y_i * f_alpha_beta_neg_y_11; + float e_dot_f_12 = e_x_i * f_alpha_beta_neg_x_12 + e_y_i * f_alpha_beta_neg_y_12; + float e_dot_f_13 = e_x_i * f_alpha_beta_neg_x_13 + e_y_i * f_alpha_beta_neg_y_13; + float e_dot_f_14 = e_x_i * f_alpha_beta_neg_x_14 + e_y_i * f_alpha_beta_neg_y_14; + float e_dot_f_15 = e_x_i * f_alpha_beta_neg_x_15 + e_y_i * f_alpha_beta_neg_y_15; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + float e_dot_f_sq_2 = e_dot_f_2 * e_dot_f_2; + float e_dot_f_sq_3 = e_dot_f_3 * e_dot_f_3; + float e_dot_f_sq_4 = e_dot_f_4 * e_dot_f_4; + float e_dot_f_sq_5 = e_dot_f_5 * e_dot_f_5; + float e_dot_f_sq_6 = e_dot_f_6 * e_dot_f_6; + float e_dot_f_sq_7 = e_dot_f_7 * e_dot_f_7; + float e_dot_f_sq_8 = e_dot_f_8 * e_dot_f_8; + float e_dot_f_sq_9 = e_dot_f_9 * e_dot_f_9; + float e_dot_f_sq_10 = e_dot_f_10 * e_dot_f_10; + float e_dot_f_sq_11 = e_dot_f_11 * e_dot_f_11; + float e_dot_f_sq_12 = e_dot_f_12 * e_dot_f_12; + float e_dot_f_sq_13 = e_dot_f_13 * e_dot_f_13; + float e_dot_f_sq_14 = e_dot_f_14 * e_dot_f_14; + float e_dot_f_sq_15 = e_dot_f_15 * e_dot_f_15; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + if (e_dot_f_2 >= 0.0 || e_dot_f_sq_2 <= rhs_2) + w_val_2 = dt_neg; + else + w_val_2 = c_neg_times_dt; + if (e_dot_f_3 >= 0.0 || e_dot_f_sq_3 <= rhs_3) + w_val_3 = dt_neg; + else + w_val_3 = c_neg_times_dt; + if (e_dot_f_4 >= 0.0 || e_dot_f_sq_4 <= rhs_4) + w_val_4 = dt_neg; + else + w_val_4 = c_neg_times_dt; + if (e_dot_f_5 >= 0.0 || e_dot_f_sq_5 <= rhs_5) + w_val_5 = dt_neg; + else + w_val_5 = c_neg_times_dt; + if (e_dot_f_6 >= 0.0 || e_dot_f_sq_6 <= rhs_6) + w_val_6 = dt_neg; + else + w_val_6 = c_neg_times_dt; + if (e_dot_f_7 >= 0.0 || e_dot_f_sq_7 <= rhs_7) + w_val_7 = dt_neg; + else + w_val_7 = c_neg_times_dt; + if (e_dot_f_8 >= 0.0 || e_dot_f_sq_8 <= rhs_8) + w_val_8 = dt_neg; + else + w_val_8 = c_neg_times_dt; + if (e_dot_f_9 >= 0.0 || e_dot_f_sq_9 <= rhs_9) + w_val_9 = dt_neg; + else + w_val_9 = c_neg_times_dt; + if (e_dot_f_10 >= 0.0 || e_dot_f_sq_10 <= rhs_10) + w_val_10 = dt_neg; + else + w_val_10 = c_neg_times_dt; + if (e_dot_f_11 >= 0.0 || e_dot_f_sq_11 <= rhs_11) + w_val_11 = dt_neg; + else + w_val_11 = c_neg_times_dt; + if (e_dot_f_12 >= 0.0 || e_dot_f_sq_12 <= rhs_12) + w_val_12 = dt_neg; + else + w_val_12 = c_neg_times_dt; + if (e_dot_f_13 >= 0.0 || e_dot_f_sq_13 <= rhs_13) + w_val_13 = dt_neg; + else + w_val_13 = c_neg_times_dt; + if (e_dot_f_14 >= 0.0 || e_dot_f_sq_14 <= rhs_14) + w_val_14 = dt_neg; + else + w_val_14 = c_neg_times_dt; + if (e_dot_f_15 >= 0.0 || e_dot_f_sq_15 <= rhs_15) + w_val_15 = dt_neg; + else + w_val_15 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_x_2 += f_alpha_beta_neg_x_2 * w_val_2; + acc_x_3 += f_alpha_beta_neg_x_3 * w_val_3; + acc_x_4 += f_alpha_beta_neg_x_4 * w_val_4; + acc_x_5 += f_alpha_beta_neg_x_5 * w_val_5; + acc_x_6 += f_alpha_beta_neg_x_6 * w_val_6; + acc_x_7 += f_alpha_beta_neg_x_7 * w_val_7; + acc_x_8 += f_alpha_beta_neg_x_8 * w_val_8; + acc_x_9 += f_alpha_beta_neg_x_9 * w_val_9; + acc_x_10 += f_alpha_beta_neg_x_10 * w_val_10; + acc_x_11 += f_alpha_beta_neg_x_11 * w_val_11; + acc_x_12 += f_alpha_beta_neg_x_12 * w_val_12; + acc_x_13 += f_alpha_beta_neg_x_13 * w_val_13; + acc_x_14 += f_alpha_beta_neg_x_14 * w_val_14; + acc_x_15 += f_alpha_beta_neg_x_15 * w_val_15; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + acc_y_2 += f_alpha_beta_neg_y_2 * w_val_2; + acc_y_3 += f_alpha_beta_neg_y_3 * w_val_3; + acc_y_4 += f_alpha_beta_neg_y_4 * w_val_4; + acc_y_5 += f_alpha_beta_neg_y_5 * w_val_5; + acc_y_6 += f_alpha_beta_neg_y_6 * w_val_6; + acc_y_7 += f_alpha_beta_neg_y_7 * w_val_7; + acc_y_8 += f_alpha_beta_neg_y_8 * w_val_8; + acc_y_9 += f_alpha_beta_neg_y_9 * w_val_9; + acc_y_10 += f_alpha_beta_neg_y_10 * w_val_10; + acc_y_11 += f_alpha_beta_neg_y_11 * w_val_11; + acc_y_12 += f_alpha_beta_neg_y_12 * w_val_12; + acc_y_13 += f_alpha_beta_neg_y_13 * w_val_13; + acc_y_14 += f_alpha_beta_neg_y_14 * w_val_14; + acc_y_15 += f_alpha_beta_neg_y_15 * w_val_15; + } + + for (; j < n_p; j++) { + float r_ab_x = r_ab_x_lut[i * n_p + j]; + float r_ab_y = r_ab_y_lut[i * n_p + j]; + float r_ab_norm = r_ab_norm_lut[i * n_p + j]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1 + acc_x_2 + acc_x_3 + acc_x_4 + acc_x_5 + acc_x_6 + acc_x_7 + acc_x_8 + acc_x_9 + acc_x_10 + acc_x_11 + acc_x_12 + acc_x_13 + acc_x_14 + acc_x_15; + f_alpha_dt_y += acc_y_0 + acc_y_1 + acc_y_2 + acc_y_3 + acc_y_4 + acc_y_5 + acc_y_6 + acc_y_7 + acc_y_8 + acc_y_9 + acc_y_10 + acc_y_11 + acc_y_12 + acc_y_13 + acc_y_14 + acc_y_15; + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val; + + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f >= cos_phi) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + + float w_norm = sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + + float w_norm2 = w_norm > EPS ? w_norm : EPS; + + float correction_factor = min(1.0f, v_max[i] / w_norm2); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5)) { + pos_x[i] = 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] == 1 && pos_x[i] < 1.5) { + pos_x[i] = BOARD_LENGTH - 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + float ex = destination_x[i] - pos_x[i]; + float ey = destination_y[i] - pos_y[i]; + + float e_norm = sqrt(ex * ex + ey * ey); + float e_norm2 = e_norm > EPS ? e_norm : EPS; + float e_norm_inv = 1.0 / e_norm2; + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + float velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + + float velocity_norm = sqrt(velocity_norm_sq); + + sb_sq[i] = velocity_norm_sq * delta_t_sq; + + sb_e_x[i] = e_x[i] * (velocity_norm * delta_t); + sb_e_y[i] = e_y[i] * (velocity_norm * delta_t); + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_symmetry_ILP_2.cpp b/variants/variant_float_inlining_SoA_symmetry_ILP_2.cpp new file mode 100644 index 0000000..89d55f4 --- /dev/null +++ b/variants/variant_float_inlining_SoA_symmetry_ILP_2.cpp @@ -0,0 +1,572 @@ +// # Changes: Added ILP for j-loop with K = 2 +// # Predecessor: variant_float_inlining_SoA_symmetry.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +vector pos_x, pos_y; +vector sb_sq; +vector sb_e_x, sb_e_y; +vector e_x, e_y; +vector curr_vel_x, curr_vel_y; +vector w_x, w_y; +vector desired_v0; +vector v_max; +vector destination_x, destination_y; +vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + pos_x.resize(n_p); + pos_y.resize(n_p); + sb_sq.resize(n_p); + sb_e_x.resize(n_p); + sb_e_y.resize(n_p); + e_x.resize(n_p); + e_y.resize(n_p); + curr_vel_x.resize(n_p); + curr_vel_y.resize(n_p); + w_x.resize(n_p); + w_y.resize(n_p); + desired_v0.resize(n_p); + v_max.resize(n_p); + destination_x.resize(n_p); + destination_y.resize(n_p); + color.resize(n_p); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + vector r_ab_x_lut(n_p * n_p, 0.0); + vector r_ab_y_lut(n_p * n_p, 0.0); + vector r_ab_norm_lut(n_p * n_p, 0.0); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + float e_x_i = e_x[i]; + float e_y_i = e_y[i]; + float pos_x_i = pos_x[i]; + float pos_y_i = pos_y[i]; + + for (int j = i + 1; j < n_p; j++) { + float dx = pos_x_i - pos_x[j]; + float dy = pos_y_i - pos_y[j]; + float norm = sqrt(dx * dx + dy * dy); + int idx = i * n_p + j; + r_ab_x_lut[idx] = dx; + r_ab_y_lut[idx] = dy; + r_ab_norm_lut[idx] = norm; + } + + int j; + + float acc_x_0 = 0.0f, acc_x_1 = 0.0f; + float acc_y_0 = 0.0f, acc_y_1 = 0.0f; + + for (j = 0; j + 2 < i; j += 2) { + float r_ab_x_0 = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_x_1 = -1 * r_ab_x_lut[(j + 1) * n_p + i]; + + float r_ab_y_0 = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_y_1 = -1 * r_ab_y_lut[(j + 1) * n_p + i]; + + float r_ab_norm_0 = r_ab_norm_lut[j * n_p + i]; + float r_ab_norm_1 = r_ab_norm_lut[(j + 1) * n_p + i]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + + float w_val_0, w_val_1; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + } + + for (; j < i; j++) { + float r_ab_x = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_y = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_norm = r_ab_norm_lut[j * n_p + i]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1; + f_alpha_dt_y += acc_y_0 + acc_y_1; + + acc_x_0 = 0.0f; acc_x_1 = 0.0f; + acc_y_0 = 0.0f; acc_y_1 = 0.0f; + + for (j = i + 1; j + 2 < n_p; j += 2) { + float r_ab_x_0 = r_ab_x_lut[i * n_p + j]; + float r_ab_x_1 = r_ab_x_lut[i * n_p + j + 1]; + + float r_ab_y_0 = r_ab_y_lut[i * n_p + j]; + float r_ab_y_1 = r_ab_y_lut[i * n_p + j + 1]; + + float r_ab_norm_0 = r_ab_norm_lut[i * n_p + j]; + float r_ab_norm_1 = r_ab_norm_lut[i * n_p + j + 1]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + + float w_val_0, w_val_1; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + } + + for (; j < n_p; j++) { + float r_ab_x = r_ab_x_lut[i * n_p + j]; + float r_ab_y = r_ab_y_lut[i * n_p + j]; + float r_ab_norm = r_ab_norm_lut[i * n_p + j]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1; + f_alpha_dt_y += acc_y_0 + acc_y_1; + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val; + + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f >= cos_phi) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + float w_norm = sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + + float w_norm2 = w_norm > EPS ? w_norm : EPS; + + float correction_factor = min(1.0f, v_max[i] / w_norm2); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5)) { + pos_x[i] = 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] == 1 && pos_x[i] < 1.5) { + pos_x[i] = BOARD_LENGTH - 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + float ex = destination_x[i] - pos_x[i]; + float ey = destination_y[i] - pos_y[i]; + + float e_norm = sqrt(ex * ex + ey * ey); + float e_norm2 = e_norm > EPS ? e_norm : EPS; + float e_norm_inv = 1.0 / e_norm2; + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + float velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + + float velocity_norm = sqrt(velocity_norm_sq); + + sb_sq[i] = velocity_norm_sq * delta_t_sq; + + sb_e_x[i] = e_x[i] * (velocity_norm * delta_t); + sb_e_y[i] = e_y[i] * (velocity_norm * delta_t); + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_symmetry_ILP_4.cpp b/variants/variant_float_inlining_SoA_symmetry_ILP_4.cpp new file mode 100644 index 0000000..3637448 --- /dev/null +++ b/variants/variant_float_inlining_SoA_symmetry_ILP_4.cpp @@ -0,0 +1,685 @@ +// # Changes: Added ILP for j-loop with K = 4 +// # Predecessor: variant_float_inlining_SoA_symmetry_ILP_2.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +vector pos_x, pos_y; +vector sb_sq; +vector sb_e_x, sb_e_y; +vector e_x, e_y; +vector curr_vel_x, curr_vel_y; +vector w_x, w_y; +vector desired_v0; +vector v_max; +vector destination_x, destination_y; +vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + pos_x.resize(n_p); + pos_y.resize(n_p); + sb_sq.resize(n_p); + sb_e_x.resize(n_p); + sb_e_y.resize(n_p); + e_x.resize(n_p); + e_y.resize(n_p); + curr_vel_x.resize(n_p); + curr_vel_y.resize(n_p); + w_x.resize(n_p); + w_y.resize(n_p); + desired_v0.resize(n_p); + v_max.resize(n_p); + destination_x.resize(n_p); + destination_y.resize(n_p); + color.resize(n_p); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + vector r_ab_x_lut(n_p * n_p, 0.0); + vector r_ab_y_lut(n_p * n_p, 0.0); + vector r_ab_norm_lut(n_p * n_p, 0.0); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + float e_x_i = e_x[i]; + float e_y_i = e_y[i]; + float pos_x_i = pos_x[i]; + float pos_y_i = pos_y[i]; + + for (int j = i + 1; j < n_p; j++) { + float dx = pos_x_i - pos_x[j]; + float dy = pos_y_i - pos_y[j]; + float norm = sqrt(dx * dx + dy * dy); + int idx = i * n_p + j; + r_ab_x_lut[idx] = dx; + r_ab_y_lut[idx] = dy; + r_ab_norm_lut[idx] = norm; + } + + int j; + + float acc_x_0 = 0.0f, acc_x_1 = 0.0f, acc_x_2 = 0.0f, acc_x_3 = 0.0f; + float acc_y_0 = 0.0f, acc_y_1 = 0.0f, acc_y_2 = 0.0f, acc_y_3 = 0.0f; + + for (j = 0; j + 4 < i; j += 4) { + float r_ab_x_0 = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_x_1 = -1 * r_ab_x_lut[(j + 1) * n_p + i]; + float r_ab_x_2 = -1 * r_ab_x_lut[(j + 2) * n_p + i]; + float r_ab_x_3 = -1 * r_ab_x_lut[(j + 3) * n_p + i]; + + float r_ab_y_0 = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_y_1 = -1 * r_ab_y_lut[(j + 1) * n_p + i]; + float r_ab_y_2 = -1 * r_ab_y_lut[(j + 2) * n_p + i]; + float r_ab_y_3 = -1 * r_ab_y_lut[(j + 3) * n_p + i]; + + float r_ab_norm_0 = r_ab_norm_lut[j * n_p + i]; + float r_ab_norm_1 = r_ab_norm_lut[(j + 1) * n_p + i]; + float r_ab_norm_2 = r_ab_norm_lut[(j + 2) * n_p + i]; + float r_ab_norm_3 = r_ab_norm_lut[(j + 3) * n_p + i]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + r_ab_norm_2 = r_ab_norm_2 > EPS ? r_ab_norm_2 : EPS; + r_ab_norm_3 = r_ab_norm_3 > EPS ? r_ab_norm_3 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + float r_ab2_x_2 = r_ab_x_2 - sb_e_x[j + 2]; + float r_ab2_x_3 = r_ab_x_3 - sb_e_x[j + 3]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + float r_ab2_y_2 = r_ab_y_2 - sb_e_y[j + 2]; + float r_ab2_y_3 = r_ab_y_3 - sb_e_y[j + 3]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + float r_ab2_norm_2 = sqrt(r_ab2_x_2 * r_ab2_x_2 + r_ab2_y_2 * r_ab2_y_2); + float r_ab2_norm_3 = sqrt(r_ab2_x_3 * r_ab2_x_3 + r_ab2_y_3 * r_ab2_y_3); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + r_ab2_norm_2 = r_ab2_norm_2 > EPS ? r_ab2_norm_2 : EPS; + r_ab2_norm_3 = r_ab2_norm_3 > EPS ? r_ab2_norm_3 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + float r_ab_norm_sum_2 = r_ab_norm_2 + r_ab2_norm_2; + float r_ab_norm_sum_3 = r_ab_norm_3 + r_ab2_norm_3; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + float two_b_sq_2 = r_ab_norm_sum_2 * r_ab_norm_sum_2 - sb_sq[j + 2]; + float two_b_sq_3 = r_ab_norm_sum_3 * r_ab_norm_sum_3 - sb_sq[j + 3]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + two_b_sq_2 = two_b_sq_2 > EPS_SQ ? two_b_sq_2 : EPS_SQ; + two_b_sq_3 = two_b_sq_3 > EPS_SQ ? two_b_sq_3 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + float two_b_2 = sqrt(two_b_sq_2); + float two_b_3 = sqrt(two_b_sq_3); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_2 = (r_ab_norm_sum_2 / (two_b_2 * r_ab_norm_2 * r_ab2_norm_2)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_2 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_3 = (r_ab_norm_sum_3 / (two_b_3 * r_ab_norm_3 * r_ab2_norm_3)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_3 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + float scalar_mult_r_ab_2 = scalar_mult_common_2 * r_ab2_norm_2; + float scalar_mult_r_ab_3 = scalar_mult_common_3 * r_ab2_norm_3; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + float scalar_mult_r_ab2_2 = scalar_mult_common_2 * r_ab_norm_2; + float scalar_mult_r_ab2_3 = scalar_mult_common_3 * r_ab_norm_3; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_x_2 = r_ab_x_2 * scalar_mult_r_ab_2 + r_ab2_x_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_x_3 = r_ab_x_3 * scalar_mult_r_ab_3 + r_ab2_x_3 * scalar_mult_r_ab2_3; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_y_2 = r_ab_y_2 * scalar_mult_r_ab_2 + r_ab2_y_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_y_3 = r_ab_y_3 * scalar_mult_r_ab_3 + r_ab2_y_3 * scalar_mult_r_ab2_3; + + float w_val_0, w_val_1, w_val_2, w_val_3; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + float f_norm_sq_2 = f_alpha_beta_neg_x_2 * f_alpha_beta_neg_x_2 + f_alpha_beta_neg_y_2 * f_alpha_beta_neg_y_2; + float f_norm_sq_3 = f_alpha_beta_neg_x_3 * f_alpha_beta_neg_x_3 + f_alpha_beta_neg_y_3 * f_alpha_beta_neg_y_3; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + float rhs_2 = f_norm_sq_2 * cos_phi_sq; + float rhs_3 = f_norm_sq_3 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + float e_dot_f_2 = e_x_i * f_alpha_beta_neg_x_2 + e_y_i * f_alpha_beta_neg_y_2; + float e_dot_f_3 = e_x_i * f_alpha_beta_neg_x_3 + e_y_i * f_alpha_beta_neg_y_3; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + float e_dot_f_sq_2 = e_dot_f_2 * e_dot_f_2; + float e_dot_f_sq_3 = e_dot_f_3 * e_dot_f_3; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + if (e_dot_f_2 >= 0.0 || e_dot_f_sq_2 <= rhs_2) + w_val_2 = dt_neg; + else + w_val_2 = c_neg_times_dt; + if (e_dot_f_3 >= 0.0 || e_dot_f_sq_3 <= rhs_3) + w_val_3 = dt_neg; + else + w_val_3 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_x_2 += f_alpha_beta_neg_x_2 * w_val_2; + acc_x_3 += f_alpha_beta_neg_x_3 * w_val_3; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + acc_y_2 += f_alpha_beta_neg_y_2 * w_val_2; + acc_y_3 += f_alpha_beta_neg_y_3 * w_val_3; + } + + for (; j < i; j++) { + float r_ab_x = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_y = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_norm = r_ab_norm_lut[j * n_p + i]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1 + acc_x_2 + acc_x_3; + f_alpha_dt_y += acc_y_0 + acc_y_1 + acc_y_2 + acc_y_3; + + acc_x_0 = 0.0f; + acc_x_1 = 0.0f; + acc_x_2 = 0.0f; + acc_x_3 = 0.0f; + acc_y_0 = 0.0f; + acc_y_1 = 0.0f; + acc_y_2 = 0.0f; + acc_y_3 = 0.0f; + + for (j = i + 1; j + 4 < n_p; j += 4) { + float r_ab_x_0 = r_ab_x_lut[i * n_p + j]; + float r_ab_x_1 = r_ab_x_lut[i * n_p + j + 1]; + float r_ab_x_2 = r_ab_x_lut[i * n_p + j + 2]; + float r_ab_x_3 = r_ab_x_lut[i * n_p + j + 3]; + + float r_ab_y_0 = r_ab_y_lut[i * n_p + j]; + float r_ab_y_1 = r_ab_y_lut[i * n_p + j + 1]; + float r_ab_y_2 = r_ab_y_lut[i * n_p + j + 2]; + float r_ab_y_3 = r_ab_y_lut[i * n_p + j + 3]; + + float r_ab_norm_0 = r_ab_norm_lut[i * n_p + j]; + float r_ab_norm_1 = r_ab_norm_lut[i * n_p + j + 1]; + float r_ab_norm_2 = r_ab_norm_lut[i * n_p + j + 2]; + float r_ab_norm_3 = r_ab_norm_lut[i * n_p + j + 3]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + r_ab_norm_2 = r_ab_norm_2 > EPS ? r_ab_norm_2 : EPS; + r_ab_norm_3 = r_ab_norm_3 > EPS ? r_ab_norm_3 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + float r_ab2_x_2 = r_ab_x_2 - sb_e_x[j + 2]; + float r_ab2_x_3 = r_ab_x_3 - sb_e_x[j + 3]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + float r_ab2_y_2 = r_ab_y_2 - sb_e_y[j + 2]; + float r_ab2_y_3 = r_ab_y_3 - sb_e_y[j + 3]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + float r_ab2_norm_2 = sqrt(r_ab2_x_2 * r_ab2_x_2 + r_ab2_y_2 * r_ab2_y_2); + float r_ab2_norm_3 = sqrt(r_ab2_x_3 * r_ab2_x_3 + r_ab2_y_3 * r_ab2_y_3); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + r_ab2_norm_2 = r_ab2_norm_2 > EPS ? r_ab2_norm_2 : EPS; + r_ab2_norm_3 = r_ab2_norm_3 > EPS ? r_ab2_norm_3 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + float r_ab_norm_sum_2 = r_ab_norm_2 + r_ab2_norm_2; + float r_ab_norm_sum_3 = r_ab_norm_3 + r_ab2_norm_3; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + float two_b_sq_2 = r_ab_norm_sum_2 * r_ab_norm_sum_2 - sb_sq[j + 2]; + float two_b_sq_3 = r_ab_norm_sum_3 * r_ab_norm_sum_3 - sb_sq[j + 3]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + two_b_sq_2 = two_b_sq_2 > EPS_SQ ? two_b_sq_2 : EPS_SQ; + two_b_sq_3 = two_b_sq_3 > EPS_SQ ? two_b_sq_3 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + float two_b_2 = sqrt(two_b_sq_2); + float two_b_3 = sqrt(two_b_sq_3); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_2 = (r_ab_norm_sum_2 / (two_b_2 * r_ab_norm_2 * r_ab2_norm_2)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_2 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_3 = (r_ab_norm_sum_3 / (two_b_3 * r_ab_norm_3 * r_ab2_norm_3)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_3 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + float scalar_mult_r_ab_2 = scalar_mult_common_2 * r_ab2_norm_2; + float scalar_mult_r_ab_3 = scalar_mult_common_3 * r_ab2_norm_3; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + float scalar_mult_r_ab2_2 = scalar_mult_common_2 * r_ab_norm_2; + float scalar_mult_r_ab2_3 = scalar_mult_common_3 * r_ab_norm_3; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_x_2 = r_ab_x_2 * scalar_mult_r_ab_2 + r_ab2_x_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_x_3 = r_ab_x_3 * scalar_mult_r_ab_3 + r_ab2_x_3 * scalar_mult_r_ab2_3; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_y_2 = r_ab_y_2 * scalar_mult_r_ab_2 + r_ab2_y_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_y_3 = r_ab_y_3 * scalar_mult_r_ab_3 + r_ab2_y_3 * scalar_mult_r_ab2_3; + + float w_val_0, w_val_1, w_val_2, w_val_3; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + float f_norm_sq_2 = f_alpha_beta_neg_x_2 * f_alpha_beta_neg_x_2 + f_alpha_beta_neg_y_2 * f_alpha_beta_neg_y_2; + float f_norm_sq_3 = f_alpha_beta_neg_x_3 * f_alpha_beta_neg_x_3 + f_alpha_beta_neg_y_3 * f_alpha_beta_neg_y_3; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + float rhs_2 = f_norm_sq_2 * cos_phi_sq; + float rhs_3 = f_norm_sq_3 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + float e_dot_f_2 = e_x_i * f_alpha_beta_neg_x_2 + e_y_i * f_alpha_beta_neg_y_2; + float e_dot_f_3 = e_x_i * f_alpha_beta_neg_x_3 + e_y_i * f_alpha_beta_neg_y_3; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + float e_dot_f_sq_2 = e_dot_f_2 * e_dot_f_2; + float e_dot_f_sq_3 = e_dot_f_3 * e_dot_f_3; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + if (e_dot_f_2 >= 0.0 || e_dot_f_sq_2 <= rhs_2) + w_val_2 = dt_neg; + else + w_val_2 = c_neg_times_dt; + if (e_dot_f_3 >= 0.0 || e_dot_f_sq_3 <= rhs_3) + w_val_3 = dt_neg; + else + w_val_3 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_x_2 += f_alpha_beta_neg_x_2 * w_val_2; + acc_x_3 += f_alpha_beta_neg_x_3 * w_val_3; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + acc_y_2 += f_alpha_beta_neg_y_2 * w_val_2; + acc_y_3 += f_alpha_beta_neg_y_3 * w_val_3; + } + + for (; j < n_p; j++) { + float r_ab_x = r_ab_x_lut[i * n_p + j]; + float r_ab_y = r_ab_y_lut[i * n_p + j]; + float r_ab_norm = r_ab_norm_lut[i * n_p + j]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1 + acc_x_2 + acc_x_3; + f_alpha_dt_y += acc_y_0 + acc_y_1 + acc_y_2 + acc_y_3; + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val; + + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f >= cos_phi) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + + float w_norm = sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + + float w_norm2 = w_norm > EPS ? w_norm : EPS; + + float correction_factor = min(1.0f, v_max[i] / w_norm2); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5)) { + pos_x[i] = 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] == 1 && pos_x[i] < 1.5) { + pos_x[i] = BOARD_LENGTH - 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + float ex = destination_x[i] - pos_x[i]; + float ey = destination_y[i] - pos_y[i]; + + float e_norm = sqrt(ex * ex + ey * ey); + float e_norm2 = e_norm > EPS ? e_norm : EPS; + float e_norm_inv = 1.0 / e_norm2; + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + float velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + + float velocity_norm = sqrt(velocity_norm_sq); + + sb_sq[i] = velocity_norm_sq * delta_t_sq; + + sb_e_x[i] = e_x[i] * (velocity_norm * delta_t); + sb_e_y[i] = e_y[i] * (velocity_norm * delta_t); + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_symmetry_ILP_8.cpp b/variants/variant_float_inlining_SoA_symmetry_ILP_8.cpp new file mode 100644 index 0000000..31fa35b --- /dev/null +++ b/variants/variant_float_inlining_SoA_symmetry_ILP_8.cpp @@ -0,0 +1,911 @@ +// # Changes: Added ILP for j-loop with K = 8 +// # Predecessor: variant_float_inlining_SoA_symmetry_ILP_4.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +vector pos_x, pos_y; +vector sb_sq; +vector sb_e_x, sb_e_y; +vector e_x, e_y; +vector curr_vel_x, curr_vel_y; +vector w_x, w_y; +vector desired_v0; +vector v_max; +vector destination_x, destination_y; +vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + pos_x.resize(n_p); + pos_y.resize(n_p); + sb_sq.resize(n_p); + sb_e_x.resize(n_p); + sb_e_y.resize(n_p); + e_x.resize(n_p); + e_y.resize(n_p); + curr_vel_x.resize(n_p); + curr_vel_y.resize(n_p); + w_x.resize(n_p); + w_y.resize(n_p); + desired_v0.resize(n_p); + v_max.resize(n_p); + destination_x.resize(n_p); + destination_y.resize(n_p); + color.resize(n_p); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + float v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + vector r_ab_x_lut(n_p * n_p, 0.0); + vector r_ab_y_lut(n_p * n_p, 0.0); + vector r_ab_norm_lut(n_p * n_p, 0.0); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + float e_x_i = e_x[i]; + float e_y_i = e_y[i]; + float pos_x_i = pos_x[i]; + float pos_y_i = pos_y[i]; + + for (int j = i + 1; j < n_p; j++) { + float dx = pos_x_i - pos_x[j]; + float dy = pos_y_i - pos_y[j]; + float norm = sqrt(dx * dx + dy * dy); + int idx = i * n_p + j; + r_ab_x_lut[idx] = dx; + r_ab_y_lut[idx] = dy; + r_ab_norm_lut[idx] = norm; + } + + int j; + + float acc_x_0 = 0.0f, acc_x_1 = 0.0f, acc_x_2 = 0.0f, acc_x_3 = 0.0f; + float acc_x_4 = 0.0f, acc_x_5 = 0.0f, acc_x_6 = 0.0f, acc_x_7 = 0.0f; + float acc_y_0 = 0.0f, acc_y_1 = 0.0f, acc_y_2 = 0.0f, acc_y_3 = 0.0f; + float acc_y_4 = 0.0f, acc_y_5 = 0.0f, acc_y_6 = 0.0f, acc_y_7 = 0.0f; + + for (j = 0; j + 8 < i; j += 8) { + float r_ab_x_0 = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_x_1 = -1 * r_ab_x_lut[(j + 1) * n_p + i]; + float r_ab_x_2 = -1 * r_ab_x_lut[(j + 2) * n_p + i]; + float r_ab_x_3 = -1 * r_ab_x_lut[(j + 3) * n_p + i]; + float r_ab_x_4 = -1 * r_ab_x_lut[(j + 4) * n_p + i]; + float r_ab_x_5 = -1 * r_ab_x_lut[(j + 5) * n_p + i]; + float r_ab_x_6 = -1 * r_ab_x_lut[(j + 6) * n_p + i]; + float r_ab_x_7 = -1 * r_ab_x_lut[(j + 7) * n_p + i]; + + float r_ab_y_0 = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_y_1 = -1 * r_ab_y_lut[(j + 1) * n_p + i]; + float r_ab_y_2 = -1 * r_ab_y_lut[(j + 2) * n_p + i]; + float r_ab_y_3 = -1 * r_ab_y_lut[(j + 3) * n_p + i]; + float r_ab_y_4 = -1 * r_ab_y_lut[(j + 4) * n_p + i]; + float r_ab_y_5 = -1 * r_ab_y_lut[(j + 5) * n_p + i]; + float r_ab_y_6 = -1 * r_ab_y_lut[(j + 6) * n_p + i]; + float r_ab_y_7 = -1 * r_ab_y_lut[(j + 7) * n_p + i]; + + float r_ab_norm_0 = r_ab_norm_lut[j * n_p + i]; + float r_ab_norm_1 = r_ab_norm_lut[(j + 1) * n_p + i]; + float r_ab_norm_2 = r_ab_norm_lut[(j + 2) * n_p + i]; + float r_ab_norm_3 = r_ab_norm_lut[(j + 3) * n_p + i]; + float r_ab_norm_4 = r_ab_norm_lut[(j + 4) * n_p + i]; + float r_ab_norm_5 = r_ab_norm_lut[(j + 5) * n_p + i]; + float r_ab_norm_6 = r_ab_norm_lut[(j + 6) * n_p + i]; + float r_ab_norm_7 = r_ab_norm_lut[(j + 7) * n_p + i]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + r_ab_norm_2 = r_ab_norm_2 > EPS ? r_ab_norm_2 : EPS; + r_ab_norm_3 = r_ab_norm_3 > EPS ? r_ab_norm_3 : EPS; + r_ab_norm_4 = r_ab_norm_4 > EPS ? r_ab_norm_4 : EPS; + r_ab_norm_5 = r_ab_norm_5 > EPS ? r_ab_norm_5 : EPS; + r_ab_norm_6 = r_ab_norm_6 > EPS ? r_ab_norm_6 : EPS; + r_ab_norm_7 = r_ab_norm_7 > EPS ? r_ab_norm_7 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + float r_ab2_x_2 = r_ab_x_2 - sb_e_x[j + 2]; + float r_ab2_x_3 = r_ab_x_3 - sb_e_x[j + 3]; + float r_ab2_x_4 = r_ab_x_4 - sb_e_x[j + 4]; + float r_ab2_x_5 = r_ab_x_5 - sb_e_x[j + 5]; + float r_ab2_x_6 = r_ab_x_6 - sb_e_x[j + 6]; + float r_ab2_x_7 = r_ab_x_7 - sb_e_x[j + 7]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + float r_ab2_y_2 = r_ab_y_2 - sb_e_y[j + 2]; + float r_ab2_y_3 = r_ab_y_3 - sb_e_y[j + 3]; + float r_ab2_y_4 = r_ab_y_4 - sb_e_y[j + 4]; + float r_ab2_y_5 = r_ab_y_5 - sb_e_y[j + 5]; + float r_ab2_y_6 = r_ab_y_6 - sb_e_y[j + 6]; + float r_ab2_y_7 = r_ab_y_7 - sb_e_y[j + 7]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + float r_ab2_norm_2 = sqrt(r_ab2_x_2 * r_ab2_x_2 + r_ab2_y_2 * r_ab2_y_2); + float r_ab2_norm_3 = sqrt(r_ab2_x_3 * r_ab2_x_3 + r_ab2_y_3 * r_ab2_y_3); + float r_ab2_norm_4 = sqrt(r_ab2_x_4 * r_ab2_x_4 + r_ab2_y_4 * r_ab2_y_4); + float r_ab2_norm_5 = sqrt(r_ab2_x_5 * r_ab2_x_5 + r_ab2_y_5 * r_ab2_y_5); + float r_ab2_norm_6 = sqrt(r_ab2_x_6 * r_ab2_x_6 + r_ab2_y_6 * r_ab2_y_6); + float r_ab2_norm_7 = sqrt(r_ab2_x_7 * r_ab2_x_7 + r_ab2_y_7 * r_ab2_y_7); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + r_ab2_norm_2 = r_ab2_norm_2 > EPS ? r_ab2_norm_2 : EPS; + r_ab2_norm_3 = r_ab2_norm_3 > EPS ? r_ab2_norm_3 : EPS; + r_ab2_norm_4 = r_ab2_norm_4 > EPS ? r_ab2_norm_4 : EPS; + r_ab2_norm_5 = r_ab2_norm_5 > EPS ? r_ab2_norm_5 : EPS; + r_ab2_norm_6 = r_ab2_norm_6 > EPS ? r_ab2_norm_6 : EPS; + r_ab2_norm_7 = r_ab2_norm_7 > EPS ? r_ab2_norm_7 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + float r_ab_norm_sum_2 = r_ab_norm_2 + r_ab2_norm_2; + float r_ab_norm_sum_3 = r_ab_norm_3 + r_ab2_norm_3; + float r_ab_norm_sum_4 = r_ab_norm_4 + r_ab2_norm_4; + float r_ab_norm_sum_5 = r_ab_norm_5 + r_ab2_norm_5; + float r_ab_norm_sum_6 = r_ab_norm_6 + r_ab2_norm_6; + float r_ab_norm_sum_7 = r_ab_norm_7 + r_ab2_norm_7; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + float two_b_sq_2 = r_ab_norm_sum_2 * r_ab_norm_sum_2 - sb_sq[j + 2]; + float two_b_sq_3 = r_ab_norm_sum_3 * r_ab_norm_sum_3 - sb_sq[j + 3]; + float two_b_sq_4 = r_ab_norm_sum_4 * r_ab_norm_sum_4 - sb_sq[j + 4]; + float two_b_sq_5 = r_ab_norm_sum_5 * r_ab_norm_sum_5 - sb_sq[j + 5]; + float two_b_sq_6 = r_ab_norm_sum_6 * r_ab_norm_sum_6 - sb_sq[j + 6]; + float two_b_sq_7 = r_ab_norm_sum_7 * r_ab_norm_sum_7 - sb_sq[j + 7]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + two_b_sq_2 = two_b_sq_2 > EPS_SQ ? two_b_sq_2 : EPS_SQ; + two_b_sq_3 = two_b_sq_3 > EPS_SQ ? two_b_sq_3 : EPS_SQ; + two_b_sq_4 = two_b_sq_4 > EPS_SQ ? two_b_sq_4 : EPS_SQ; + two_b_sq_5 = two_b_sq_5 > EPS_SQ ? two_b_sq_5 : EPS_SQ; + two_b_sq_6 = two_b_sq_6 > EPS_SQ ? two_b_sq_6 : EPS_SQ; + two_b_sq_7 = two_b_sq_7 > EPS_SQ ? two_b_sq_7 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + float two_b_2 = sqrt(two_b_sq_2); + float two_b_3 = sqrt(two_b_sq_3); + float two_b_4 = sqrt(two_b_sq_4); + float two_b_5 = sqrt(two_b_sq_5); + float two_b_6 = sqrt(two_b_sq_6); + float two_b_7 = sqrt(two_b_sq_7); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_2 = (r_ab_norm_sum_2 / (two_b_2 * r_ab_norm_2 * r_ab2_norm_2)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_2 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_3 = (r_ab_norm_sum_3 / (two_b_3 * r_ab_norm_3 * r_ab2_norm_3)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_3 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_4 = (r_ab_norm_sum_4 / (two_b_4 * r_ab_norm_4 * r_ab2_norm_4)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_4 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_5 = (r_ab_norm_sum_5 / (two_b_5 * r_ab_norm_5 * r_ab2_norm_5)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_5 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_6 = (r_ab_norm_sum_6 / (two_b_6 * r_ab_norm_6 * r_ab2_norm_6)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_6 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_7 = (r_ab_norm_sum_7 / (two_b_7 * r_ab_norm_7 * r_ab2_norm_7)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_7 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + float scalar_mult_r_ab_2 = scalar_mult_common_2 * r_ab2_norm_2; + float scalar_mult_r_ab_3 = scalar_mult_common_3 * r_ab2_norm_3; + float scalar_mult_r_ab_4 = scalar_mult_common_4 * r_ab2_norm_4; + float scalar_mult_r_ab_5 = scalar_mult_common_5 * r_ab2_norm_5; + float scalar_mult_r_ab_6 = scalar_mult_common_6 * r_ab2_norm_6; + float scalar_mult_r_ab_7 = scalar_mult_common_7 * r_ab2_norm_7; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + float scalar_mult_r_ab2_2 = scalar_mult_common_2 * r_ab_norm_2; + float scalar_mult_r_ab2_3 = scalar_mult_common_3 * r_ab_norm_3; + float scalar_mult_r_ab2_4 = scalar_mult_common_4 * r_ab_norm_4; + float scalar_mult_r_ab2_5 = scalar_mult_common_5 * r_ab_norm_5; + float scalar_mult_r_ab2_6 = scalar_mult_common_6 * r_ab_norm_6; + float scalar_mult_r_ab2_7 = scalar_mult_common_7 * r_ab_norm_7; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_x_2 = r_ab_x_2 * scalar_mult_r_ab_2 + r_ab2_x_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_x_3 = r_ab_x_3 * scalar_mult_r_ab_3 + r_ab2_x_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_x_4 = r_ab_x_4 * scalar_mult_r_ab_4 + r_ab2_x_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_x_5 = r_ab_x_5 * scalar_mult_r_ab_5 + r_ab2_x_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_x_6 = r_ab_x_6 * scalar_mult_r_ab_6 + r_ab2_x_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_x_7 = r_ab_x_7 * scalar_mult_r_ab_7 + r_ab2_x_7 * scalar_mult_r_ab2_7; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_y_2 = r_ab_y_2 * scalar_mult_r_ab_2 + r_ab2_y_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_y_3 = r_ab_y_3 * scalar_mult_r_ab_3 + r_ab2_y_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_y_4 = r_ab_y_4 * scalar_mult_r_ab_4 + r_ab2_y_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_y_5 = r_ab_y_5 * scalar_mult_r_ab_5 + r_ab2_y_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_y_6 = r_ab_y_6 * scalar_mult_r_ab_6 + r_ab2_y_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_y_7 = r_ab_y_7 * scalar_mult_r_ab_7 + r_ab2_y_7 * scalar_mult_r_ab2_7; + + float w_val_0, w_val_1, w_val_2, w_val_3, w_val_4, w_val_5, w_val_6, w_val_7; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + float f_norm_sq_2 = f_alpha_beta_neg_x_2 * f_alpha_beta_neg_x_2 + f_alpha_beta_neg_y_2 * f_alpha_beta_neg_y_2; + float f_norm_sq_3 = f_alpha_beta_neg_x_3 * f_alpha_beta_neg_x_3 + f_alpha_beta_neg_y_3 * f_alpha_beta_neg_y_3; + float f_norm_sq_4 = f_alpha_beta_neg_x_4 * f_alpha_beta_neg_x_4 + f_alpha_beta_neg_y_4 * f_alpha_beta_neg_y_4; + float f_norm_sq_5 = f_alpha_beta_neg_x_5 * f_alpha_beta_neg_x_5 + f_alpha_beta_neg_y_5 * f_alpha_beta_neg_y_5; + float f_norm_sq_6 = f_alpha_beta_neg_x_6 * f_alpha_beta_neg_x_6 + f_alpha_beta_neg_y_6 * f_alpha_beta_neg_y_6; + float f_norm_sq_7 = f_alpha_beta_neg_x_7 * f_alpha_beta_neg_x_7 + f_alpha_beta_neg_y_7 * f_alpha_beta_neg_y_7; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + float rhs_2 = f_norm_sq_2 * cos_phi_sq; + float rhs_3 = f_norm_sq_3 * cos_phi_sq; + float rhs_4 = f_norm_sq_4 * cos_phi_sq; + float rhs_5 = f_norm_sq_5 * cos_phi_sq; + float rhs_6 = f_norm_sq_6 * cos_phi_sq; + float rhs_7 = f_norm_sq_7 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + float e_dot_f_2 = e_x_i * f_alpha_beta_neg_x_2 + e_y_i * f_alpha_beta_neg_y_2; + float e_dot_f_3 = e_x_i * f_alpha_beta_neg_x_3 + e_y_i * f_alpha_beta_neg_y_3; + float e_dot_f_4 = e_x_i * f_alpha_beta_neg_x_4 + e_y_i * f_alpha_beta_neg_y_4; + float e_dot_f_5 = e_x_i * f_alpha_beta_neg_x_5 + e_y_i * f_alpha_beta_neg_y_5; + float e_dot_f_6 = e_x_i * f_alpha_beta_neg_x_6 + e_y_i * f_alpha_beta_neg_y_6; + float e_dot_f_7 = e_x_i * f_alpha_beta_neg_x_7 + e_y_i * f_alpha_beta_neg_y_7; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + float e_dot_f_sq_2 = e_dot_f_2 * e_dot_f_2; + float e_dot_f_sq_3 = e_dot_f_3 * e_dot_f_3; + float e_dot_f_sq_4 = e_dot_f_4 * e_dot_f_4; + float e_dot_f_sq_5 = e_dot_f_5 * e_dot_f_5; + float e_dot_f_sq_6 = e_dot_f_6 * e_dot_f_6; + float e_dot_f_sq_7 = e_dot_f_7 * e_dot_f_7; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + if (e_dot_f_2 >= 0.0 || e_dot_f_sq_2 <= rhs_2) + w_val_2 = dt_neg; + else + w_val_2 = c_neg_times_dt; + if (e_dot_f_3 >= 0.0 || e_dot_f_sq_3 <= rhs_3) + w_val_3 = dt_neg; + else + w_val_3 = c_neg_times_dt; + if (e_dot_f_4 >= 0.0 || e_dot_f_sq_4 <= rhs_4) + w_val_4 = dt_neg; + else + w_val_4 = c_neg_times_dt; + if (e_dot_f_5 >= 0.0 || e_dot_f_sq_5 <= rhs_5) + w_val_5 = dt_neg; + else + w_val_5 = c_neg_times_dt; + if (e_dot_f_6 >= 0.0 || e_dot_f_sq_6 <= rhs_6) + w_val_6 = dt_neg; + else + w_val_6 = c_neg_times_dt; + if (e_dot_f_7 >= 0.0 || e_dot_f_sq_7 <= rhs_7) + w_val_7 = dt_neg; + else + w_val_7 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_x_2 += f_alpha_beta_neg_x_2 * w_val_2; + acc_x_3 += f_alpha_beta_neg_x_3 * w_val_3; + acc_x_4 += f_alpha_beta_neg_x_4 * w_val_4; + acc_x_5 += f_alpha_beta_neg_x_5 * w_val_5; + acc_x_6 += f_alpha_beta_neg_x_6 * w_val_6; + acc_x_7 += f_alpha_beta_neg_x_7 * w_val_7; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + acc_y_2 += f_alpha_beta_neg_y_2 * w_val_2; + acc_y_3 += f_alpha_beta_neg_y_3 * w_val_3; + acc_y_4 += f_alpha_beta_neg_y_4 * w_val_4; + acc_y_5 += f_alpha_beta_neg_y_5 * w_val_5; + acc_y_6 += f_alpha_beta_neg_y_6 * w_val_6; + acc_y_7 += f_alpha_beta_neg_y_7 * w_val_7; + } + + for (; j < i; j++) { + float r_ab_x = -1 * r_ab_x_lut[j * n_p + i]; + float r_ab_y = -1 * r_ab_y_lut[j * n_p + i]; + float r_ab_norm = r_ab_norm_lut[j * n_p + i]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1 + acc_x_2 + acc_x_3 + acc_x_4 + acc_x_5 + acc_x_6 + acc_x_7; + f_alpha_dt_y += acc_y_0 + acc_y_1 + acc_y_2 + acc_y_3 + acc_y_4 + acc_y_5 + acc_y_6 + acc_y_7; + + acc_x_0 = 0.0f; + acc_x_1 = 0.0f; + acc_x_2 = 0.0f; + acc_x_3 = 0.0f; + acc_x_4 = 0.0f; + acc_x_5 = 0.0f; + acc_x_6 = 0.0f; + acc_x_7 = 0.0f; + acc_y_0 = 0.0f; + acc_y_1 = 0.0f; + acc_y_2 = 0.0f; + acc_y_3 = 0.0f; + acc_y_4 = 0.0f; + acc_y_5 = 0.0f; + acc_y_6 = 0.0f; + acc_y_7 = 0.0f; + + for (j = i + 1; j + 8 < n_p; j += 8) { + float r_ab_x_0 = r_ab_x_lut[i * n_p + j]; + float r_ab_x_1 = r_ab_x_lut[i * n_p + j + 1]; + float r_ab_x_2 = r_ab_x_lut[i * n_p + j + 2]; + float r_ab_x_3 = r_ab_x_lut[i * n_p + j + 3]; + float r_ab_x_4 = r_ab_x_lut[i * n_p + j + 4]; + float r_ab_x_5 = r_ab_x_lut[i * n_p + j + 5]; + float r_ab_x_6 = r_ab_x_lut[i * n_p + j + 6]; + float r_ab_x_7 = r_ab_x_lut[i * n_p + j + 7]; + + float r_ab_y_0 = r_ab_y_lut[i * n_p + j]; + float r_ab_y_1 = r_ab_y_lut[i * n_p + j + 1]; + float r_ab_y_2 = r_ab_y_lut[i * n_p + j + 2]; + float r_ab_y_3 = r_ab_y_lut[i * n_p + j + 3]; + float r_ab_y_4 = r_ab_y_lut[i * n_p + j + 4]; + float r_ab_y_5 = r_ab_y_lut[i * n_p + j + 5]; + float r_ab_y_6 = r_ab_y_lut[i * n_p + j + 6]; + float r_ab_y_7 = r_ab_y_lut[i * n_p + j + 7]; + + float r_ab_norm_0 = r_ab_norm_lut[i * n_p + j]; + float r_ab_norm_1 = r_ab_norm_lut[i * n_p + j + 1]; + float r_ab_norm_2 = r_ab_norm_lut[i * n_p + j + 2]; + float r_ab_norm_3 = r_ab_norm_lut[i * n_p + j + 3]; + float r_ab_norm_4 = r_ab_norm_lut[i * n_p + j + 4]; + float r_ab_norm_5 = r_ab_norm_lut[i * n_p + j + 5]; + float r_ab_norm_6 = r_ab_norm_lut[i * n_p + j + 6]; + float r_ab_norm_7 = r_ab_norm_lut[i * n_p + j + 7]; + + r_ab_norm_0 = r_ab_norm_0 > EPS ? r_ab_norm_0 : EPS; + r_ab_norm_1 = r_ab_norm_1 > EPS ? r_ab_norm_1 : EPS; + r_ab_norm_2 = r_ab_norm_2 > EPS ? r_ab_norm_2 : EPS; + r_ab_norm_3 = r_ab_norm_3 > EPS ? r_ab_norm_3 : EPS; + r_ab_norm_4 = r_ab_norm_4 > EPS ? r_ab_norm_4 : EPS; + r_ab_norm_5 = r_ab_norm_5 > EPS ? r_ab_norm_5 : EPS; + r_ab_norm_6 = r_ab_norm_6 > EPS ? r_ab_norm_6 : EPS; + r_ab_norm_7 = r_ab_norm_7 > EPS ? r_ab_norm_7 : EPS; + + float r_ab2_x_0 = r_ab_x_0 - sb_e_x[j]; + float r_ab2_x_1 = r_ab_x_1 - sb_e_x[j + 1]; + float r_ab2_x_2 = r_ab_x_2 - sb_e_x[j + 2]; + float r_ab2_x_3 = r_ab_x_3 - sb_e_x[j + 3]; + float r_ab2_x_4 = r_ab_x_4 - sb_e_x[j + 4]; + float r_ab2_x_5 = r_ab_x_5 - sb_e_x[j + 5]; + float r_ab2_x_6 = r_ab_x_6 - sb_e_x[j + 6]; + float r_ab2_x_7 = r_ab_x_7 - sb_e_x[j + 7]; + + float r_ab2_y_0 = r_ab_y_0 - sb_e_y[j]; + float r_ab2_y_1 = r_ab_y_1 - sb_e_y[j + 1]; + float r_ab2_y_2 = r_ab_y_2 - sb_e_y[j + 2]; + float r_ab2_y_3 = r_ab_y_3 - sb_e_y[j + 3]; + float r_ab2_y_4 = r_ab_y_4 - sb_e_y[j + 4]; + float r_ab2_y_5 = r_ab_y_5 - sb_e_y[j + 5]; + float r_ab2_y_6 = r_ab_y_6 - sb_e_y[j + 6]; + float r_ab2_y_7 = r_ab_y_7 - sb_e_y[j + 7]; + + float r_ab2_norm_0 = sqrt(r_ab2_x_0 * r_ab2_x_0 + r_ab2_y_0 * r_ab2_y_0); + float r_ab2_norm_1 = sqrt(r_ab2_x_1 * r_ab2_x_1 + r_ab2_y_1 * r_ab2_y_1); + float r_ab2_norm_2 = sqrt(r_ab2_x_2 * r_ab2_x_2 + r_ab2_y_2 * r_ab2_y_2); + float r_ab2_norm_3 = sqrt(r_ab2_x_3 * r_ab2_x_3 + r_ab2_y_3 * r_ab2_y_3); + float r_ab2_norm_4 = sqrt(r_ab2_x_4 * r_ab2_x_4 + r_ab2_y_4 * r_ab2_y_4); + float r_ab2_norm_5 = sqrt(r_ab2_x_5 * r_ab2_x_5 + r_ab2_y_5 * r_ab2_y_5); + float r_ab2_norm_6 = sqrt(r_ab2_x_6 * r_ab2_x_6 + r_ab2_y_6 * r_ab2_y_6); + float r_ab2_norm_7 = sqrt(r_ab2_x_7 * r_ab2_x_7 + r_ab2_y_7 * r_ab2_y_7); + + r_ab2_norm_0 = r_ab2_norm_0 > EPS ? r_ab2_norm_0 : EPS; + r_ab2_norm_1 = r_ab2_norm_1 > EPS ? r_ab2_norm_1 : EPS; + r_ab2_norm_2 = r_ab2_norm_2 > EPS ? r_ab2_norm_2 : EPS; + r_ab2_norm_3 = r_ab2_norm_3 > EPS ? r_ab2_norm_3 : EPS; + r_ab2_norm_4 = r_ab2_norm_4 > EPS ? r_ab2_norm_4 : EPS; + r_ab2_norm_5 = r_ab2_norm_5 > EPS ? r_ab2_norm_5 : EPS; + r_ab2_norm_6 = r_ab2_norm_6 > EPS ? r_ab2_norm_6 : EPS; + r_ab2_norm_7 = r_ab2_norm_7 > EPS ? r_ab2_norm_7 : EPS; + + float r_ab_norm_sum_0 = r_ab_norm_0 + r_ab2_norm_0; + float r_ab_norm_sum_1 = r_ab_norm_1 + r_ab2_norm_1; + float r_ab_norm_sum_2 = r_ab_norm_2 + r_ab2_norm_2; + float r_ab_norm_sum_3 = r_ab_norm_3 + r_ab2_norm_3; + float r_ab_norm_sum_4 = r_ab_norm_4 + r_ab2_norm_4; + float r_ab_norm_sum_5 = r_ab_norm_5 + r_ab2_norm_5; + float r_ab_norm_sum_6 = r_ab_norm_6 + r_ab2_norm_6; + float r_ab_norm_sum_7 = r_ab_norm_7 + r_ab2_norm_7; + + float two_b_sq_0 = r_ab_norm_sum_0 * r_ab_norm_sum_0 - sb_sq[j]; + float two_b_sq_1 = r_ab_norm_sum_1 * r_ab_norm_sum_1 - sb_sq[j + 1]; + float two_b_sq_2 = r_ab_norm_sum_2 * r_ab_norm_sum_2 - sb_sq[j + 2]; + float two_b_sq_3 = r_ab_norm_sum_3 * r_ab_norm_sum_3 - sb_sq[j + 3]; + float two_b_sq_4 = r_ab_norm_sum_4 * r_ab_norm_sum_4 - sb_sq[j + 4]; + float two_b_sq_5 = r_ab_norm_sum_5 * r_ab_norm_sum_5 - sb_sq[j + 5]; + float two_b_sq_6 = r_ab_norm_sum_6 * r_ab_norm_sum_6 - sb_sq[j + 6]; + float two_b_sq_7 = r_ab_norm_sum_7 * r_ab_norm_sum_7 - sb_sq[j + 7]; + + two_b_sq_0 = two_b_sq_0 > EPS_SQ ? two_b_sq_0 : EPS_SQ; + two_b_sq_1 = two_b_sq_1 > EPS_SQ ? two_b_sq_1 : EPS_SQ; + two_b_sq_2 = two_b_sq_2 > EPS_SQ ? two_b_sq_2 : EPS_SQ; + two_b_sq_3 = two_b_sq_3 > EPS_SQ ? two_b_sq_3 : EPS_SQ; + two_b_sq_4 = two_b_sq_4 > EPS_SQ ? two_b_sq_4 : EPS_SQ; + two_b_sq_5 = two_b_sq_5 > EPS_SQ ? two_b_sq_5 : EPS_SQ; + two_b_sq_6 = two_b_sq_6 > EPS_SQ ? two_b_sq_6 : EPS_SQ; + two_b_sq_7 = two_b_sq_7 > EPS_SQ ? two_b_sq_7 : EPS_SQ; + + float two_b_0 = sqrt(two_b_sq_0); + float two_b_1 = sqrt(two_b_sq_1); + float two_b_2 = sqrt(two_b_sq_2); + float two_b_3 = sqrt(two_b_sq_3); + float two_b_4 = sqrt(two_b_sq_4); + float two_b_5 = sqrt(two_b_sq_5); + float two_b_6 = sqrt(two_b_sq_6); + float two_b_7 = sqrt(two_b_sq_7); + + float scalar_mult_common_0 = (r_ab_norm_sum_0 / (two_b_0 * r_ab_norm_0 * r_ab2_norm_0)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_0 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_1 = (r_ab_norm_sum_1 / (two_b_1 * r_ab_norm_1 * r_ab2_norm_1)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_1 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_2 = (r_ab_norm_sum_2 / (two_b_2 * r_ab_norm_2 * r_ab2_norm_2)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_2 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_3 = (r_ab_norm_sum_3 / (two_b_3 * r_ab_norm_3 * r_ab2_norm_3)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_3 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_4 = (r_ab_norm_sum_4 / (two_b_4 * r_ab_norm_4 * r_ab2_norm_4)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_4 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_5 = (r_ab_norm_sum_5 / (two_b_5 * r_ab_norm_5 * r_ab2_norm_5)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_5 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_6 = (r_ab_norm_sum_6 / (two_b_6 * r_ab_norm_6 * r_ab2_norm_6)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_6 * NEG_TWO_SIGMA_INV)); + float scalar_mult_common_7 = (r_ab_norm_sum_7 / (two_b_7 * r_ab_norm_7 * r_ab2_norm_7)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b_7 * NEG_TWO_SIGMA_INV)); + + float scalar_mult_r_ab_0 = scalar_mult_common_0 * r_ab2_norm_0; + float scalar_mult_r_ab_1 = scalar_mult_common_1 * r_ab2_norm_1; + float scalar_mult_r_ab_2 = scalar_mult_common_2 * r_ab2_norm_2; + float scalar_mult_r_ab_3 = scalar_mult_common_3 * r_ab2_norm_3; + float scalar_mult_r_ab_4 = scalar_mult_common_4 * r_ab2_norm_4; + float scalar_mult_r_ab_5 = scalar_mult_common_5 * r_ab2_norm_5; + float scalar_mult_r_ab_6 = scalar_mult_common_6 * r_ab2_norm_6; + float scalar_mult_r_ab_7 = scalar_mult_common_7 * r_ab2_norm_7; + + float scalar_mult_r_ab2_0 = scalar_mult_common_0 * r_ab_norm_0; + float scalar_mult_r_ab2_1 = scalar_mult_common_1 * r_ab_norm_1; + float scalar_mult_r_ab2_2 = scalar_mult_common_2 * r_ab_norm_2; + float scalar_mult_r_ab2_3 = scalar_mult_common_3 * r_ab_norm_3; + float scalar_mult_r_ab2_4 = scalar_mult_common_4 * r_ab_norm_4; + float scalar_mult_r_ab2_5 = scalar_mult_common_5 * r_ab_norm_5; + float scalar_mult_r_ab2_6 = scalar_mult_common_6 * r_ab_norm_6; + float scalar_mult_r_ab2_7 = scalar_mult_common_7 * r_ab_norm_7; + + float f_alpha_beta_neg_x_0 = r_ab_x_0 * scalar_mult_r_ab_0 + r_ab2_x_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_x_1 = r_ab_x_1 * scalar_mult_r_ab_1 + r_ab2_x_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_x_2 = r_ab_x_2 * scalar_mult_r_ab_2 + r_ab2_x_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_x_3 = r_ab_x_3 * scalar_mult_r_ab_3 + r_ab2_x_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_x_4 = r_ab_x_4 * scalar_mult_r_ab_4 + r_ab2_x_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_x_5 = r_ab_x_5 * scalar_mult_r_ab_5 + r_ab2_x_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_x_6 = r_ab_x_6 * scalar_mult_r_ab_6 + r_ab2_x_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_x_7 = r_ab_x_7 * scalar_mult_r_ab_7 + r_ab2_x_7 * scalar_mult_r_ab2_7; + + float f_alpha_beta_neg_y_0 = r_ab_y_0 * scalar_mult_r_ab_0 + r_ab2_y_0 * scalar_mult_r_ab2_0; + float f_alpha_beta_neg_y_1 = r_ab_y_1 * scalar_mult_r_ab_1 + r_ab2_y_1 * scalar_mult_r_ab2_1; + float f_alpha_beta_neg_y_2 = r_ab_y_2 * scalar_mult_r_ab_2 + r_ab2_y_2 * scalar_mult_r_ab2_2; + float f_alpha_beta_neg_y_3 = r_ab_y_3 * scalar_mult_r_ab_3 + r_ab2_y_3 * scalar_mult_r_ab2_3; + float f_alpha_beta_neg_y_4 = r_ab_y_4 * scalar_mult_r_ab_4 + r_ab2_y_4 * scalar_mult_r_ab2_4; + float f_alpha_beta_neg_y_5 = r_ab_y_5 * scalar_mult_r_ab_5 + r_ab2_y_5 * scalar_mult_r_ab2_5; + float f_alpha_beta_neg_y_6 = r_ab_y_6 * scalar_mult_r_ab_6 + r_ab2_y_6 * scalar_mult_r_ab2_6; + float f_alpha_beta_neg_y_7 = r_ab_y_7 * scalar_mult_r_ab_7 + r_ab2_y_7 * scalar_mult_r_ab2_7; + + float w_val_0, w_val_1, w_val_2, w_val_3, w_val_4, w_val_5, w_val_6, w_val_7; + + float f_norm_sq_0 = f_alpha_beta_neg_x_0 * f_alpha_beta_neg_x_0 + f_alpha_beta_neg_y_0 * f_alpha_beta_neg_y_0; + float f_norm_sq_1 = f_alpha_beta_neg_x_1 * f_alpha_beta_neg_x_1 + f_alpha_beta_neg_y_1 * f_alpha_beta_neg_y_1; + float f_norm_sq_2 = f_alpha_beta_neg_x_2 * f_alpha_beta_neg_x_2 + f_alpha_beta_neg_y_2 * f_alpha_beta_neg_y_2; + float f_norm_sq_3 = f_alpha_beta_neg_x_3 * f_alpha_beta_neg_x_3 + f_alpha_beta_neg_y_3 * f_alpha_beta_neg_y_3; + float f_norm_sq_4 = f_alpha_beta_neg_x_4 * f_alpha_beta_neg_x_4 + f_alpha_beta_neg_y_4 * f_alpha_beta_neg_y_4; + float f_norm_sq_5 = f_alpha_beta_neg_x_5 * f_alpha_beta_neg_x_5 + f_alpha_beta_neg_y_5 * f_alpha_beta_neg_y_5; + float f_norm_sq_6 = f_alpha_beta_neg_x_6 * f_alpha_beta_neg_x_6 + f_alpha_beta_neg_y_6 * f_alpha_beta_neg_y_6; + float f_norm_sq_7 = f_alpha_beta_neg_x_7 * f_alpha_beta_neg_x_7 + f_alpha_beta_neg_y_7 * f_alpha_beta_neg_y_7; + + float rhs_0 = f_norm_sq_0 * cos_phi_sq; + float rhs_1 = f_norm_sq_1 * cos_phi_sq; + float rhs_2 = f_norm_sq_2 * cos_phi_sq; + float rhs_3 = f_norm_sq_3 * cos_phi_sq; + float rhs_4 = f_norm_sq_4 * cos_phi_sq; + float rhs_5 = f_norm_sq_5 * cos_phi_sq; + float rhs_6 = f_norm_sq_6 * cos_phi_sq; + float rhs_7 = f_norm_sq_7 * cos_phi_sq; + + float e_dot_f_0 = e_x_i * f_alpha_beta_neg_x_0 + e_y_i * f_alpha_beta_neg_y_0; + float e_dot_f_1 = e_x_i * f_alpha_beta_neg_x_1 + e_y_i * f_alpha_beta_neg_y_1; + float e_dot_f_2 = e_x_i * f_alpha_beta_neg_x_2 + e_y_i * f_alpha_beta_neg_y_2; + float e_dot_f_3 = e_x_i * f_alpha_beta_neg_x_3 + e_y_i * f_alpha_beta_neg_y_3; + float e_dot_f_4 = e_x_i * f_alpha_beta_neg_x_4 + e_y_i * f_alpha_beta_neg_y_4; + float e_dot_f_5 = e_x_i * f_alpha_beta_neg_x_5 + e_y_i * f_alpha_beta_neg_y_5; + float e_dot_f_6 = e_x_i * f_alpha_beta_neg_x_6 + e_y_i * f_alpha_beta_neg_y_6; + float e_dot_f_7 = e_x_i * f_alpha_beta_neg_x_7 + e_y_i * f_alpha_beta_neg_y_7; + + float e_dot_f_sq_0 = e_dot_f_0 * e_dot_f_0; + float e_dot_f_sq_1 = e_dot_f_1 * e_dot_f_1; + float e_dot_f_sq_2 = e_dot_f_2 * e_dot_f_2; + float e_dot_f_sq_3 = e_dot_f_3 * e_dot_f_3; + float e_dot_f_sq_4 = e_dot_f_4 * e_dot_f_4; + float e_dot_f_sq_5 = e_dot_f_5 * e_dot_f_5; + float e_dot_f_sq_6 = e_dot_f_6 * e_dot_f_6; + float e_dot_f_sq_7 = e_dot_f_7 * e_dot_f_7; + + if (e_dot_f_0 >= 0.0 || e_dot_f_sq_0 <= rhs_0) + w_val_0 = dt_neg; + else + w_val_0 = c_neg_times_dt; + if (e_dot_f_1 >= 0.0 || e_dot_f_sq_1 <= rhs_1) + w_val_1 = dt_neg; + else + w_val_1 = c_neg_times_dt; + if (e_dot_f_2 >= 0.0 || e_dot_f_sq_2 <= rhs_2) + w_val_2 = dt_neg; + else + w_val_2 = c_neg_times_dt; + if (e_dot_f_3 >= 0.0 || e_dot_f_sq_3 <= rhs_3) + w_val_3 = dt_neg; + else + w_val_3 = c_neg_times_dt; + if (e_dot_f_4 >= 0.0 || e_dot_f_sq_4 <= rhs_4) + w_val_4 = dt_neg; + else + w_val_4 = c_neg_times_dt; + if (e_dot_f_5 >= 0.0 || e_dot_f_sq_5 <= rhs_5) + w_val_5 = dt_neg; + else + w_val_5 = c_neg_times_dt; + if (e_dot_f_6 >= 0.0 || e_dot_f_sq_6 <= rhs_6) + w_val_6 = dt_neg; + else + w_val_6 = c_neg_times_dt; + if (e_dot_f_7 >= 0.0 || e_dot_f_sq_7 <= rhs_7) + w_val_7 = dt_neg; + else + w_val_7 = c_neg_times_dt; + + acc_x_0 += f_alpha_beta_neg_x_0 * w_val_0; + acc_x_1 += f_alpha_beta_neg_x_1 * w_val_1; + acc_x_2 += f_alpha_beta_neg_x_2 * w_val_2; + acc_x_3 += f_alpha_beta_neg_x_3 * w_val_3; + acc_x_4 += f_alpha_beta_neg_x_4 * w_val_4; + acc_x_5 += f_alpha_beta_neg_x_5 * w_val_5; + acc_x_6 += f_alpha_beta_neg_x_6 * w_val_6; + acc_x_7 += f_alpha_beta_neg_x_7 * w_val_7; + acc_y_0 += f_alpha_beta_neg_y_0 * w_val_0; + acc_y_1 += f_alpha_beta_neg_y_1 * w_val_1; + acc_y_2 += f_alpha_beta_neg_y_2 * w_val_2; + acc_y_3 += f_alpha_beta_neg_y_3 * w_val_3; + acc_y_4 += f_alpha_beta_neg_y_4 * w_val_4; + acc_y_5 += f_alpha_beta_neg_y_5 * w_val_5; + acc_y_6 += f_alpha_beta_neg_y_6 * w_val_6; + acc_y_7 += f_alpha_beta_neg_y_7 * w_val_7; + } + + for (; j < n_p; j++) { + float r_ab_x = r_ab_x_lut[i * n_p + j]; + float r_ab_y = r_ab_y_lut[i * n_p + j]; + float r_ab_norm = r_ab_norm_lut[i * n_p + j]; + + r_ab_norm = r_ab_norm > EPS ? r_ab_norm : EPS; + + float r_ab2_x = r_ab_x - sb_e_x[j]; + float r_ab2_y = r_ab_y - sb_e_y[j]; + + float r_ab2_norm = sqrt(r_ab2_x * r_ab2_x + r_ab2_y * r_ab2_y); + r_ab2_norm = r_ab2_norm > EPS ? r_ab2_norm : EPS; + + float r_ab_norm_sum = r_ab_norm + r_ab2_norm; + + float two_b_sq = r_ab_norm_sum * r_ab_norm_sum - sb_sq[j]; + two_b_sq = two_b_sq > EPS_SQ ? two_b_sq : EPS_SQ; + float two_b = sqrt(two_b_sq); + float scalar_mult_common = (r_ab_norm_sum / (two_b * r_ab_norm * r_ab2_norm)) * (NEG_V0_AB_div_TWO_SIGMA * exp(two_b * NEG_TWO_SIGMA_INV)); + float scalar_mult_r_ab = scalar_mult_common * r_ab2_norm; + float scalar_mult_r_ab2 = scalar_mult_common * r_ab_norm; + + float f_alpha_beta_neg_x = r_ab_x * scalar_mult_r_ab + r_ab2_x * scalar_mult_r_ab2; + float f_alpha_beta_neg_y = r_ab_y * scalar_mult_r_ab + r_ab2_y * scalar_mult_r_ab2; + + float w_val; + + float f_norm_sq = f_alpha_beta_neg_x * f_alpha_beta_neg_x + f_alpha_beta_neg_y * f_alpha_beta_neg_y; + float rhs = f_norm_sq * cos_phi_sq; + float e_dot_f = e_x[i] * f_alpha_beta_neg_x + e_y[i] * f_alpha_beta_neg_y; + float e_dot_f_sq = e_dot_f * e_dot_f; + if (e_dot_f >= 0.0 || e_dot_f_sq <= rhs) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_beta_neg_x * w_val; + f_alpha_dt_y += f_alpha_beta_neg_y * w_val; + } + + f_alpha_dt_x += acc_x_0 + acc_x_1 + acc_x_2 + acc_x_3 + acc_x_4 + acc_x_5 + acc_x_6 + acc_x_7; + f_alpha_dt_y += acc_y_0 + acc_y_1 + acc_y_2 + acc_y_3 + acc_y_4 + acc_y_5 + acc_y_6 + acc_y_7; + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val; + + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f >= cos_phi) + w_val = dt_neg; + else + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + + float w_norm = sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + + float w_norm2 = w_norm > EPS ? w_norm : EPS; + + float correction_factor = min(1.0f, v_max[i] / w_norm2); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5)) { + pos_x[i] = 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] == 1 && pos_x[i] < 1.5) { + pos_x[i] = BOARD_LENGTH - 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + float ex = destination_x[i] - pos_x[i]; + float ey = destination_y[i] - pos_y[i]; + + float e_norm = sqrt(ex * ex + ey * ey); + float e_norm2 = e_norm > EPS ? e_norm : EPS; + float e_norm_inv = 1.0 / e_norm2; + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + float velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + + float velocity_norm = sqrt(velocity_norm_sq); + + sb_sq[i] = velocity_norm_sq * delta_t_sq; + + sb_e_x[i] = e_x[i] * (velocity_norm * delta_t); + sb_e_y[i] = e_y[i] * (velocity_norm * delta_t); + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp index 758896e..2164224 100644 --- a/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp +++ b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp @@ -1,7 +1,6 @@ // # Changes: // # - Ported AVX2 to AVX-512 // # - Replaced Obstacle with the version that contains no Vector2d -// # - 64-byte aligned the vectors + replaced loadu with load // # - Implemented Fast Inverse Square Root // # Predecessor: variant_inlining_7_SoA_AVX2_polyexp.cpp @@ -292,11 +291,11 @@ int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, c __mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ); __mmask8 active = m_neq_i & m_in_range; - __m512d pjx = _mm512_load_pd(&pos_x[j]); - __m512d pjy = _mm512_load_pd(&pos_y[j]); - __m512d sbe_jx = _mm512_load_pd(&sb_e_x[j]); - __m512d sbe_jy = _mm512_load_pd(&sb_e_y[j]); - __m512d sb_jsq = _mm512_load_pd(&sb_sq[j]); + __m512d pjx = _mm512_loadu_pd(&pos_x[j]); + __m512d pjy = _mm512_loadu_pd(&pos_y[j]); + __m512d sbe_jx = _mm512_loadu_pd(&sb_e_x[j]); + __m512d sbe_jy = _mm512_loadu_pd(&sb_e_y[j]); + __m512d sb_jsq = _mm512_loadu_pd(&sb_sq[j]); __m512d r_ab_x = _mm512_sub_pd(pix, pjx); __m512d r_ab_y = _mm512_sub_pd(piy, pjy); diff --git a/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp index 0651a99..6f5b8df 100644 --- a/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp +++ b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp @@ -288,11 +288,11 @@ int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, c __mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ); __mmask8 active = m_neq_i & m_in_range; - __m512d pjx = _mm512_load_pd(&pos_x[j]); - __m512d pjy = _mm512_load_pd(&pos_y[j]); - __m512d sbe_jx = _mm512_load_pd(&sb_e_x[j]); - __m512d sbe_jy = _mm512_load_pd(&sb_e_y[j]); - __m512d sb_jsq = _mm512_load_pd(&sb_sq[j]); + __m512d pjx = _mm512_loadu_pd(&pos_x[j]); + __m512d pjy = _mm512_loadu_pd(&pos_y[j]); + __m512d sbe_jx = _mm512_loadu_pd(&sb_e_x[j]); + __m512d sbe_jy = _mm512_loadu_pd(&sb_e_y[j]); + __m512d sb_jsq = _mm512_loadu_pd(&sb_sq[j]); __m512d r_ab_x = _mm512_sub_pd(pix, pjx); __m512d r_ab_y = _mm512_sub_pd(piy, pjy); diff --git a/variants/variant_inlining_8_SoA_AVX512_polyexp_moreavx.cpp b/variants/variant_inlining_8_SoA_AVX512_polyexp_moreavx.cpp new file mode 100644 index 0000000..ceab1cb --- /dev/null +++ b/variants/variant_inlining_8_SoA_AVX512_polyexp_moreavx.cpp @@ -0,0 +1,453 @@ +// # Changes: Vectorised second pass with AVX-512 +// # Predecessor: variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp + +#include "../headers/board_reduced.hpp" +#include "../headers/constants.hpp" +#include "../headers/pedestrian_reduced.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0 +#define BOARD_LENGTH 50.0 +#define WALL_EPS 1e-3 +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999 + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { + +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 7) & ~7; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + double x = x_gen_left(gen); + double y = y_gen(gen); + + double dest_x = BOARD_LENGTH + 30.0; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int i = n_p_half; i < n_p; i++) { + double x = x_gen_right(gen); + double y = y_gen(gen); + + double dest_x = -30; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9; + pos_y[j] = 1e9; + sb_sq[j] = 0.0; + sb_e_x[j] = 0.0; + sb_e_y[j] = 0.0; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512d _avx512_exp_pd(__m512d x) { + + const __m512d log2e = _mm512_set1_pd(1.4426950408889634); + __m512d nf = _mm512_roundscale_pd(_mm512_mul_pd(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + const __m512d ln2 = _mm512_set1_pd(0.6931471805599453); + __m512d r = _mm512_fnmadd_pd(nf, ln2, x); + + const __m512d c7 = _mm512_set1_pd(1.0 / 5040.0); + const __m512d c6 = _mm512_set1_pd(1.0 / 720.0); + const __m512d c5 = _mm512_set1_pd(1.0 / 120.0); + const __m512d c4 = _mm512_set1_pd(1.0 / 24.0); + const __m512d c3 = _mm512_set1_pd(1.0 / 6.0); + const __m512d c2 = _mm512_set1_pd(0.5); + const __m512d c1 = _mm512_set1_pd(1.0); + const __m512d c0 = _mm512_set1_pd(1.0); + + __m512d p = c7; + p = _mm512_fmadd_pd(p, r, c6); + p = _mm512_fmadd_pd(p, r, c5); + p = _mm512_fmadd_pd(p, r, c4); + p = _mm512_fmadd_pd(p, r, c3); + p = _mm512_fmadd_pd(p, r, c2); + p = _mm512_fmadd_pd(p, r, c1); + p = _mm512_fmadd_pd(p, r, c0); + + __m256d nf_lo = _mm512_castpd512_pd256(nf); + __m256d nf_hi = _mm512_extractf64x4_pd(nf, 1); + + __m128i ni32_lo = _mm256_cvtpd_epi32(nf_lo); + __m128i ni32_hi = _mm256_cvtpd_epi32(nf_hi); + + __m256i ni64_lo = _mm256_cvtepi32_epi64(ni32_lo); + __m256i ni64_hi = _mm256_cvtepi32_epi64(ni32_hi); + + __m512i ni64 = _mm512_inserti64x4(_mm512_castsi256_si512(ni64_lo), ni64_hi, 1); + + __m512i biased = _mm512_add_epi64(ni64, _mm512_set1_epi64(1023)); + + __m512i scaled = _mm512_slli_epi64(biased, 52); + + __m512d pow2n = _mm512_castsi512_pd(scaled); + + return _mm512_mul_pd(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 7) & ~7; + + alignas(64) constexpr double lane_offset_arr[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; + const __m512d lane_off = _mm512_load_pd(lane_offset_arr); + const __m512d v_neg_v0_div_2s = _mm512_set1_pd(NEG_V0_AB_div_TWO_SIGMA); + const __m512d v_neg_2s_inv = _mm512_set1_pd(NEG_TWO_SIGMA_INV); + const __m512d v_cos_phi_inv = _mm512_set1_pd(cos_phi_inv); + const __m512d v_dt_neg = _mm512_set1_pd(dt_neg); + const __m512d v_c_neg_dt = _mm512_set1_pd(c_neg_times_dt); + const __m512d v_zero = _mm512_setzero_pd(); + + const __m512d v_np = _mm512_set1_pd(n_p); + + for (int t = 0; t < n_iter; t++) { + + for (int i = 0; i < n_p; i++) { + double pos_x_i = pos_x[i]; + double pos_y_i = pos_y[i]; + double e_x_i = e_x[i]; + double e_y_i = e_y[i]; + double desired_v0_i = desired_v0[i]; + double curr_vel_x_i = curr_vel_x[i]; + double curr_vel_y_i = curr_vel_y[i]; + + double f_alpha_dt_x = (e_x_i * desired_v0_i - curr_vel_x_i) * dt_times_tau_alpha_inv; + double f_alpha_dt_y = (e_y_i * desired_v0_i - curr_vel_y_i) * dt_times_tau_alpha_inv; + + const __m512d pix = _mm512_set1_pd(pos_x_i); + const __m512d piy = _mm512_set1_pd(pos_y_i); + const __m512d eix = _mm512_set1_pd(e_x_i); + const __m512d eiy = _mm512_set1_pd(e_y_i); + const __m512d v_i = _mm512_set1_pd((double)i); + + __m512d acc_x = _mm512_setzero_pd(); + __m512d acc_y = _mm512_setzero_pd(); + + for (int j = 0; j < n_p_pad; j += 8) { + __m512d j_idx = _mm512_add_pd(_mm512_set1_pd((double)j), lane_off); + + __mmask8 m_neq_i = _mm512_cmp_pd_mask(j_idx, v_i, _CMP_NEQ_OQ); + __mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ); + __mmask8 active = m_neq_i & m_in_range; + + __m512d pjx = _mm512_loadu_pd(&pos_x[j]); + __m512d pjy = _mm512_loadu_pd(&pos_y[j]); + __m512d sbe_jx = _mm512_loadu_pd(&sb_e_x[j]); + __m512d sbe_jy = _mm512_loadu_pd(&sb_e_y[j]); + __m512d sb_jsq = _mm512_loadu_pd(&sb_sq[j]); + + __m512d r_ab_x = _mm512_sub_pd(pix, pjx); + __m512d r_ab_y = _mm512_sub_pd(piy, pjy); + __m512d r_ab_n2 = _mm512_fmadd_pd(r_ab_x, r_ab_x, _mm512_mul_pd(r_ab_y, r_ab_y)); + __m512d r_ab_n = _mm512_sqrt_pd(r_ab_n2); + + __m512d r_ab2_x = _mm512_sub_pd(r_ab_x, sbe_jx); + __m512d r_ab2_y = _mm512_sub_pd(r_ab_y, sbe_jy); + __m512d r_ab2_n2 = _mm512_fmadd_pd(r_ab2_x, r_ab2_x, _mm512_mul_pd(r_ab2_y, r_ab2_y)); + __m512d r_ab2_n = _mm512_sqrt_pd(r_ab2_n2); + + __m512d r_sum = _mm512_add_pd(r_ab_n, r_ab2_n); + __m512d two_b_sq = _mm512_fmsub_pd(r_sum, r_sum, sb_jsq); + __m512d two_b = _mm512_sqrt_pd(two_b_sq); + + __m512d e_arg = _mm512_mul_pd(two_b, v_neg_2s_inv); + __m512d e_val = _avx512_exp_pd(e_arg); + + __m512d numer = _mm512_mul_pd(_mm512_mul_pd(r_sum, v_neg_v0_div_2s), e_val); + __m512d denom = _mm512_mul_pd(_mm512_mul_pd(two_b, r_ab_n), r_ab2_n); + __m512d smc = _mm512_div_pd(numer, denom); + + __m512d sm_r_ab = _mm512_mul_pd(smc, r_ab2_n); + __m512d sm_r_ab2 = _mm512_mul_pd(smc, r_ab_n); + + __m512d f_x = _mm512_fmadd_pd(r_ab2_x, sm_r_ab2, _mm512_mul_pd(r_ab_x, sm_r_ab)); + __m512d f_y = _mm512_fmadd_pd(r_ab2_y, sm_r_ab2, _mm512_mul_pd(r_ab_y, sm_r_ab)); + + __m512d a = _mm512_mul_pd(_mm512_fmadd_pd(eix, f_x, _mm512_mul_pd(eiy, f_y)), v_cos_phi_inv); + __m512d a_sq = _mm512_mul_pd(a, a); + __m512d b_sq = _mm512_fmadd_pd(f_x, f_x, _mm512_mul_pd(f_y, f_y)); + + __mmask8 k_ge = _mm512_cmp_pd_mask(a, v_zero, _CMP_GE_OQ); + __mmask8 k_gt = _mm512_cmp_pd_mask(a_sq, b_sq, _CMP_GT_OQ); + __mmask8 k_cond = _mm512_kand(k_ge, k_gt); + __m512d w_val = _mm512_mask_blend_pd(k_cond, v_dt_neg, v_c_neg_dt); + + __m512d f_x_safe = _mm512_maskz_mov_pd(active, f_x); + __m512d f_y_safe = _mm512_maskz_mov_pd(active, f_y); + acc_x = _mm512_fmadd_pd(f_x_safe, w_val, acc_x); + acc_y = _mm512_fmadd_pd(f_y_safe, w_val, acc_y); + } + + f_alpha_dt_x += _mm512_reduce_add_pd(acc_x); + f_alpha_dt_y += _mm512_reduce_add_pd(acc_y); + + for (int k = 0; k < board.n_obstacles; k++) { + + double v1_x = pos_x_i - board.obstacles[k].start_x; + double v1_y = pos_y_i - board.obstacles[k].start_y; + + double projection = (v1_x * board.obstacles[k].obstacle_v_x + v1_y * board.obstacles[k].obstacle_v_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + double r_aB_x = pos_x_i - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + double r_aB_y = pos_y_i - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + double r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + double scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + double f_alpha_B_neg_x = r_aB_x * scalar_mult; + double f_alpha_B_neg_y = r_aB_y * scalar_mult; + + double w_val = dt_neg; + double e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y_i : e_y_i); + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_B_neg_x * w_val; + f_alpha_dt_y += f_alpha_B_neg_y * w_val; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 8) { + + __m512d w_x_i_vec = _mm512_loadu_pd(&w_x[i]); + __m512d w_y_i_vec = _mm512_loadu_pd(&w_y[i]); + __m512d v_max_i_vec = _mm512_loadu_pd(&v_max[i]); + __m512d pos_x_i_vec = _mm512_loadu_pd(&pos_x[i]); + __m512d pos_y_i_vec = _mm512_loadu_pd(&pos_y[i]); + __m512d destination_y_i_vec = _mm512_loadu_pd(&destination_y[i]); + __m512d destination_x_i_vec = _mm512_loadu_pd(&destination_x[i]); + __m512d desired_v0_i_vec = _mm512_loadu_pd(&desired_v0[i]); + + __m256i color_i_vec = _mm256_loadu_si256((__m256i *)&color[i]); + + __m512d w_norm_sq = _mm512_fmadd_pd(w_x_i_vec, w_x_i_vec, _mm512_mul_pd(w_y_i_vec, w_y_i_vec)); + __m512d w_norm = _mm512_sqrt_pd(w_norm_sq); + __m512d correction_factor = _mm512_min_pd(_mm512_set1_pd(1.0), _mm512_div_pd(v_max_i_vec, w_norm)); + + __m512d curr_vel_x_i_vec = _mm512_mul_pd(w_x_i_vec, correction_factor); + __m512d curr_vel_y_i_vec = _mm512_mul_pd(w_y_i_vec, correction_factor); + + __m512d dt_vec = _mm512_set1_pd(dt); + pos_x_i_vec = _mm512_fmadd_pd(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); + pos_y_i_vec = _mm512_fmadd_pd(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); + + __mmask8 cond_1_left = _mm256_cmpeq_epi32_mask(color_i_vec, _mm256_setzero_si256()); + __mmask8 cond_1_right = _mm512_cmp_pd_mask(pos_x_i_vec, _mm512_set1_pd(BOARD_LENGTH - 1.5), _CMP_GT_OQ); + __mmask8 cond_1 = cond_1_left & cond_1_right; + + pos_x_i_vec = _mm512_mask_blend_pd(cond_1, pos_x_i_vec, _mm512_set1_pd(1.0)); + destination_y_i_vec = _mm512_mask_blend_pd(cond_1, destination_y_i_vec, pos_y_i_vec); + w_x_i_vec = _mm512_mask_blend_pd(cond_1, w_x_i_vec, desired_v0_i_vec); + w_y_i_vec = _mm512_mask_blend_pd(cond_1, w_y_i_vec, _mm512_setzero_pd()); + curr_vel_x_i_vec = _mm512_mask_blend_pd(cond_1, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_pd(cond_1, curr_vel_y_i_vec, w_y_i_vec); + + __mmask8 cond_2_left = _mm256_cmpeq_epi32_mask(color_i_vec, _mm256_set1_epi32(1)); + __mmask8 cond_2_right = _mm512_cmp_pd_mask(pos_x_i_vec, _mm512_set1_pd(1.5), _CMP_LT_OQ); + __mmask8 cond_2 = cond_2_left & cond_2_right; + + pos_x_i_vec = _mm512_mask_blend_pd(cond_2, pos_x_i_vec, _mm512_set1_pd(BOARD_LENGTH - 1.0)); + destination_y_i_vec = _mm512_mask_blend_pd(cond_2, destination_y_i_vec, pos_y_i_vec); + __m512d neg_desired_v0 = _mm512_xor_pd(desired_v0_i_vec, _mm512_set1_pd(-0.0)); + w_x_i_vec = _mm512_mask_blend_pd(cond_2, w_x_i_vec, neg_desired_v0); + w_y_i_vec = _mm512_mask_blend_pd(cond_2, w_y_i_vec, _mm512_setzero_pd()); + curr_vel_x_i_vec = _mm512_mask_blend_pd(cond_2, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_pd(cond_2, curr_vel_y_i_vec, w_y_i_vec); + + pos_y_i_vec = _mm512_max_pd(pos_y_i_vec, _mm512_set1_pd(WALL_EPS)); + pos_y_i_vec = _mm512_min_pd(pos_y_i_vec, _mm512_set1_pd(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512d ex_vec = _mm512_sub_pd(destination_x_i_vec, pos_x_i_vec); + __m512d ey_vec = _mm512_sub_pd(destination_y_i_vec, pos_y_i_vec); + __m512d e_norm_sq = _mm512_fmadd_pd(ex_vec, ex_vec, _mm512_mul_pd(ey_vec, ey_vec)); + __m512d e_norm = _mm512_sqrt_pd(e_norm_sq); + __m512d e_norm_inv = _mm512_div_pd(_mm512_set1_pd(1.0), e_norm); + __m512d e_x_i_vec = _mm512_mul_pd(ex_vec, e_norm_inv); + __m512d e_y_i_vec = _mm512_mul_pd(ey_vec, e_norm_inv); + + __m512d vel_norm_sq = _mm512_fmadd_pd(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_pd(curr_vel_y_i_vec, curr_vel_y_i_vec)); + __m512d sb_sq_i_vec = _mm512_mul_pd(vel_norm_sq, _mm512_set1_pd(delta_t_sq)); + __m512d vel_norm_dt = _mm512_sqrt_pd(sb_sq_i_vec); + + _mm512_storeu_pd(&curr_vel_x[i], curr_vel_x_i_vec); + _mm512_storeu_pd(&curr_vel_y[i], curr_vel_y_i_vec); + _mm512_storeu_pd(&w_x[i], w_x_i_vec); + _mm512_storeu_pd(&w_y[i], w_y_i_vec); + _mm512_storeu_pd(&pos_x[i], pos_x_i_vec); + _mm512_storeu_pd(&pos_y[i], pos_y_i_vec); + _mm512_storeu_pd(&e_x[i], e_x_i_vec); + _mm512_storeu_pd(&e_y[i], e_y_i_vec); + _mm512_storeu_pd(&destination_y[i], destination_y_i_vec); + _mm512_storeu_pd(&sb_sq[i], sb_sq_i_vec); + _mm512_storeu_pd(&sb_e_x[i], _mm512_mul_pd(e_x_i_vec, vel_norm_dt)); + _mm512_storeu_pd(&sb_e_y[i], _mm512_mul_pd(e_y_i_vec, vel_norm_dt)); + } +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_inlining_8_SoA_AVX512_polyexp_moreavx_ilp_2.cpp b/variants/variant_inlining_8_SoA_AVX512_polyexp_moreavx_ilp_2.cpp new file mode 100644 index 0000000..c7c1652 --- /dev/null +++ b/variants/variant_inlining_8_SoA_AVX512_polyexp_moreavx_ilp_2.cpp @@ -0,0 +1,533 @@ +// # Changes: Added ILP with k = 2 to j-loop +// # Predecessor: variant_inlining_8_SoA_AVX512_polyexp_moreavx.cpp + +#include "../headers/board_reduced.hpp" +#include "../headers/constants.hpp" +#include "../headers/pedestrian_reduced.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0 +#define BOARD_LENGTH 50.0 +#define WALL_EPS 1e-3 +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999 + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { + +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 7) & ~7; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + double x = x_gen_left(gen); + double y = y_gen(gen); + + double dest_x = BOARD_LENGTH + 30.0; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int i = n_p_half; i < n_p; i++) { + double x = x_gen_right(gen); + double y = y_gen(gen); + + double dest_x = -30; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9; + pos_y[j] = 1e9; + sb_sq[j] = 0.0; + sb_e_x[j] = 0.0; + sb_e_y[j] = 0.0; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512d _avx512_exp_pd(__m512d x) { + + const __m512d log2e = _mm512_set1_pd(1.4426950408889634); + __m512d nf = _mm512_roundscale_pd(_mm512_mul_pd(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + const __m512d ln2 = _mm512_set1_pd(0.6931471805599453); + __m512d r = _mm512_fnmadd_pd(nf, ln2, x); + + const __m512d c7 = _mm512_set1_pd(1.0 / 5040.0); + const __m512d c6 = _mm512_set1_pd(1.0 / 720.0); + const __m512d c5 = _mm512_set1_pd(1.0 / 120.0); + const __m512d c4 = _mm512_set1_pd(1.0 / 24.0); + const __m512d c3 = _mm512_set1_pd(1.0 / 6.0); + const __m512d c2 = _mm512_set1_pd(0.5); + const __m512d c1 = _mm512_set1_pd(1.0); + const __m512d c0 = _mm512_set1_pd(1.0); + + __m512d p = c7; + p = _mm512_fmadd_pd(p, r, c6); + p = _mm512_fmadd_pd(p, r, c5); + p = _mm512_fmadd_pd(p, r, c4); + p = _mm512_fmadd_pd(p, r, c3); + p = _mm512_fmadd_pd(p, r, c2); + p = _mm512_fmadd_pd(p, r, c1); + p = _mm512_fmadd_pd(p, r, c0); + + __m256d nf_lo = _mm512_castpd512_pd256(nf); + __m256d nf_hi = _mm512_extractf64x4_pd(nf, 1); + + __m128i ni32_lo = _mm256_cvtpd_epi32(nf_lo); + __m128i ni32_hi = _mm256_cvtpd_epi32(nf_hi); + + __m256i ni64_lo = _mm256_cvtepi32_epi64(ni32_lo); + __m256i ni64_hi = _mm256_cvtepi32_epi64(ni32_hi); + + __m512i ni64 = _mm512_inserti64x4(_mm512_castsi256_si512(ni64_lo), ni64_hi, 1); + + __m512i biased = _mm512_add_epi64(ni64, _mm512_set1_epi64(1023)); + + __m512i scaled = _mm512_slli_epi64(biased, 52); + + __m512d pow2n = _mm512_castsi512_pd(scaled); + + return _mm512_mul_pd(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 15) & ~15; + + alignas(64) constexpr double lane_offset_arr[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; + const __m512d lane_off = _mm512_load_pd(lane_offset_arr); + + const __m512d v_neg_v0_div_2s = _mm512_set1_pd(NEG_V0_AB_div_TWO_SIGMA); + const __m512d v_neg_2s_inv = _mm512_set1_pd(NEG_TWO_SIGMA_INV); + const __m512d v_cos_phi_sq = _mm512_set1_pd(cos_phi_sq); + const __m512d v_dt_neg = _mm512_set1_pd(dt_neg); + const __m512d v_c_neg_dt = _mm512_set1_pd(c_neg_times_dt); + const __m512d v_zero = _mm512_setzero_pd(); + const __m512d v_one = _mm512_set1_pd(1.0f); + const __m512d v_np = _mm512_set1_pd((float)n_p); + const __m512d v_exp_min = _mm512_set1_pd(-87.0f); + + for (int t = 0; t < n_iter; t++) { + + for (int i = 0; i < n_p; i++) { + double pos_x_i = pos_x[i]; + double pos_y_i = pos_y[i]; + double e_x_i = e_x[i]; + double e_y_i = e_y[i]; + double desired_v0_i = desired_v0[i]; + double curr_vel_x_i = curr_vel_x[i]; + double curr_vel_y_i = curr_vel_y[i]; + + double f_alpha_dt_x = (e_x_i * desired_v0_i - curr_vel_x_i) * dt_times_tau_alpha_inv; + double f_alpha_dt_y = (e_y_i * desired_v0_i - curr_vel_y_i) * dt_times_tau_alpha_inv; + + const __m512d pix = _mm512_set1_pd(pos_x_i); + const __m512d piy = _mm512_set1_pd(pos_y_i); + const __m512d eix = _mm512_set1_pd(e_x_i); + const __m512d eiy = _mm512_set1_pd(e_y_i); + const __m512d v_i = _mm512_set1_pd((double)i); + + __m512d acc_x_0 = _mm512_setzero_pd(); + __m512d acc_y_0 = _mm512_setzero_pd(); + __m512d acc_x_1 = _mm512_setzero_pd(); + __m512d acc_y_1 = _mm512_setzero_pd(); + + for (int j = 0; j < n_p_pad; j += 16) { + __m512d j_idx_0 = _mm512_add_pd(_mm512_set1_pd((double)j), lane_off); + __m512d j_idx_1 = _mm512_add_pd(_mm512_set1_pd((double)j + 8), lane_off); + + __mmask8 m_neq_i_0 = _mm512_cmp_pd_mask(j_idx_0, v_i, _CMP_NEQ_OQ); + __mmask8 m_neq_i_1 = _mm512_cmp_pd_mask(j_idx_1, v_i, _CMP_NEQ_OQ); + + __mmask8 m_in_range_0 = _mm512_cmp_pd_mask(j_idx_0, v_np, _CMP_LT_OQ); + __mmask8 m_in_range_1 = _mm512_cmp_pd_mask(j_idx_1, v_np, _CMP_LT_OQ); + + __mmask8 active_0 = m_neq_i_0 & m_in_range_0; + __mmask8 active_1 = m_neq_i_1 & m_in_range_1; + + __m512d pjx_0 = _mm512_loadu_pd(&pos_x[j]); + __m512d pjx_1 = _mm512_loadu_pd(&pos_x[j + 8]); + + __m512d pjy_0 = _mm512_loadu_pd(&pos_y[j]); + __m512d pjy_1 = _mm512_loadu_pd(&pos_y[j + 8]); + + __m512d sbe_jx_0 = _mm512_loadu_pd(&sb_e_x[j]); + __m512d sbe_jx_1 = _mm512_loadu_pd(&sb_e_x[j + 8]); + + __m512d sbe_jy_0 = _mm512_loadu_pd(&sb_e_y[j]); + __m512d sbe_jy_1 = _mm512_loadu_pd(&sb_e_y[j + 8]); + + __m512d sb_jsq_0 = _mm512_loadu_pd(&sb_sq[j]); + __m512d sb_jsq_1 = _mm512_loadu_pd(&sb_sq[j + 8]); + + __m512d r_ab_x_0 = _mm512_sub_pd(pix, pjx_0); + __m512d r_ab_x_1 = _mm512_sub_pd(pix, pjx_1); + + __m512d r_ab_y_0 = _mm512_sub_pd(piy, pjy_0); + __m512d r_ab_y_1 = _mm512_sub_pd(piy, pjy_1); + + __m512d r_ab_n2_0 = _mm512_fmadd_pd(r_ab_x_0, r_ab_x_0, _mm512_mul_pd(r_ab_y_0, r_ab_y_0)); + __m512d r_ab_n2_1 = _mm512_fmadd_pd(r_ab_x_1, r_ab_x_1, _mm512_mul_pd(r_ab_y_1, r_ab_y_1)); + + __m512d r_ab_n_0 = _mm512_sqrt_pd(r_ab_n2_0); + __m512d r_ab_n_1 = _mm512_sqrt_pd(r_ab_n2_1); + + __m512d r_ab2_x_0 = _mm512_sub_pd(r_ab_x_0, sbe_jx_0); + __m512d r_ab2_x_1 = _mm512_sub_pd(r_ab_x_1, sbe_jx_1); + + __m512d r_ab2_y_0 = _mm512_sub_pd(r_ab_y_0, sbe_jy_0); + __m512d r_ab2_y_1 = _mm512_sub_pd(r_ab_y_1, sbe_jy_1); + + __m512d r_ab2_n2_0 = _mm512_fmadd_pd(r_ab2_x_0, r_ab2_x_0, _mm512_mul_pd(r_ab2_y_0, r_ab2_y_0)); + __m512d r_ab2_n2_1 = _mm512_fmadd_pd(r_ab2_x_1, r_ab2_x_1, _mm512_mul_pd(r_ab2_y_1, r_ab2_y_1)); + + __m512d r_ab2_n_0 = _mm512_sqrt_pd(r_ab2_n2_0); + __m512d r_ab2_n_1 = _mm512_sqrt_pd(r_ab2_n2_1); + + __m512d r_sum_0 = _mm512_add_pd(r_ab_n_0, r_ab2_n_0); + __m512d r_sum_1 = _mm512_add_pd(r_ab_n_1, r_ab2_n_1); + + __m512d two_b_sq_0 = _mm512_fmsub_pd(r_sum_0, r_sum_0, sb_jsq_0); + __m512d two_b_sq_1 = _mm512_fmsub_pd(r_sum_1, r_sum_1, sb_jsq_1); + + __m512d two_b_0 = _mm512_sqrt_pd(two_b_sq_0); + __m512d two_b_1 = _mm512_sqrt_pd(two_b_sq_1); + + __m512d e_arg_0 = _mm512_mul_pd(two_b_0, v_neg_2s_inv); + __m512d e_arg_1 = _mm512_mul_pd(two_b_1, v_neg_2s_inv); + + e_arg_0 = _mm512_max_pd(e_arg_0, v_exp_min); + e_arg_1 = _mm512_max_pd(e_arg_1, v_exp_min); + + __m512d e_val_0 = _avx512_exp_pd(e_arg_0); + __m512d e_val_1 = _avx512_exp_pd(e_arg_1); + + __m512d numer_0 = _mm512_mul_pd(_mm512_mul_pd(r_sum_0, v_neg_v0_div_2s), e_val_0); + __m512d numer_1 = _mm512_mul_pd(_mm512_mul_pd(r_sum_1, v_neg_v0_div_2s), e_val_1); + + __m512d denom_0 = _mm512_mul_pd(_mm512_mul_pd(two_b_0, r_ab_n_0), r_ab2_n_0); + __m512d denom_1 = _mm512_mul_pd(_mm512_mul_pd(two_b_1, r_ab_n_1), r_ab2_n_1); + + __m512d denom_safe_0 = _mm512_mask_blend_pd(active_0, v_one, denom_0); + __m512d denom_safe_1 = _mm512_mask_blend_pd(active_1, v_one, denom_1); + + __m512d smc_0 = _mm512_div_pd(numer_0, denom_safe_0); + __m512d smc_1 = _mm512_div_pd(numer_1, denom_safe_1); + + __m512d sm_r_ab_0 = _mm512_mul_pd(smc_0, r_ab2_n_0); + __m512d sm_r_ab_1 = _mm512_mul_pd(smc_1, r_ab2_n_1); + + __m512d sm_r_ab2_0 = _mm512_mul_pd(smc_0, r_ab_n_0); + __m512d sm_r_ab2_1 = _mm512_mul_pd(smc_1, r_ab_n_1); + + __m512d f_x_0 = _mm512_fmadd_pd(r_ab2_x_0, sm_r_ab2_0, _mm512_mul_pd(r_ab_x_0, sm_r_ab_0)); + __m512d f_x_1 = _mm512_fmadd_pd(r_ab2_x_1, sm_r_ab2_1, _mm512_mul_pd(r_ab_x_1, sm_r_ab_1)); + + __m512d f_y_0 = _mm512_fmadd_pd(r_ab2_y_0, sm_r_ab2_0, _mm512_mul_pd(r_ab_y_0, sm_r_ab_0)); + __m512d f_y_1 = _mm512_fmadd_pd(r_ab2_y_1, sm_r_ab2_1, _mm512_mul_pd(r_ab_y_1, sm_r_ab_1)); + + __m512d f_norm_sq_0 = _mm512_fmadd_pd(f_x_0, f_x_0, _mm512_mul_pd(f_y_0, f_y_0)); + __m512d f_norm_sq_1 = _mm512_fmadd_pd(f_x_1, f_x_1, _mm512_mul_pd(f_y_1, f_y_1)); + + __m512d rhs_0 = _mm512_mul_pd(f_norm_sq_0, v_cos_phi_sq); + __m512d rhs_1 = _mm512_mul_pd(f_norm_sq_1, v_cos_phi_sq); + + __m512d e_dot_f_0 = _mm512_fmadd_pd(eix, f_x_0, _mm512_mul_pd(eiy, f_y_0)); + __m512d e_dot_f_1 = _mm512_fmadd_pd(eix, f_x_1, _mm512_mul_pd(eiy, f_y_1)); + + __m512d e_dot_f_sq_0 = _mm512_mul_pd(e_dot_f_0, e_dot_f_0); + __m512d e_dot_f_sq_1 = _mm512_mul_pd(e_dot_f_1, e_dot_f_1); + + __mmask8 c1_0 = _mm512_cmp_pd_mask(e_dot_f_0, v_zero, _CMP_LT_OQ); + __mmask8 c1_1 = _mm512_cmp_pd_mask(e_dot_f_1, v_zero, _CMP_LT_OQ); + + __mmask8 c2_0 = _mm512_cmp_pd_mask(e_dot_f_sq_0, rhs_0, _CMP_GT_OQ); + __mmask8 c2_1 = _mm512_cmp_pd_mask(e_dot_f_sq_1, rhs_1, _CMP_GT_OQ); + + __mmask8 cond_0 = c1_0 & c2_0; + __mmask8 cond_1 = c1_1 & c2_1; + + __m512d w_val_0 = _mm512_mask_blend_pd(cond_0, v_dt_neg, v_c_neg_dt); + __m512d w_val_1 = _mm512_mask_blend_pd(cond_1, v_dt_neg, v_c_neg_dt); + + __m512d f_x_safe_0 = _mm512_maskz_mov_pd(active_0, f_x_0); + __m512d f_x_safe_1 = _mm512_maskz_mov_pd(active_1, f_x_1); + + __m512d f_y_safe_0 = _mm512_maskz_mov_pd(active_0, f_y_0); + __m512d f_y_safe_1 = _mm512_maskz_mov_pd(active_1, f_y_1); + + acc_x_0 = _mm512_fmadd_pd(f_x_safe_0, w_val_0, acc_x_0); + acc_x_1 = _mm512_fmadd_pd(f_x_safe_1, w_val_1, acc_x_1); + + acc_y_0 = _mm512_fmadd_pd(f_y_safe_0, w_val_0, acc_y_0); + acc_y_1 = _mm512_fmadd_pd(f_y_safe_1, w_val_1, acc_y_1); + } + + f_alpha_dt_x += _mm512_reduce_add_pd(_mm512_add_pd(acc_x_0, acc_x_1)); + f_alpha_dt_y += _mm512_reduce_add_pd(_mm512_add_pd(acc_y_0, acc_y_1)); + + for (int k = 0; k < board.n_obstacles; k++) { + + double v1_x = pos_x_i - board.obstacles[k].start_x; + double v1_y = pos_y_i - board.obstacles[k].start_y; + + double projection = (v1_x * board.obstacles[k].obstacle_v_x + v1_y * board.obstacles[k].obstacle_v_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + double r_aB_x = pos_x_i - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + double r_aB_y = pos_y_i - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + double r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + double scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + double f_alpha_B_neg_x = r_aB_x * scalar_mult; + double f_alpha_B_neg_y = r_aB_y * scalar_mult; + + double w_val = dt_neg; + double e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y_i : e_y_i); + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_B_neg_x * w_val; + f_alpha_dt_y += f_alpha_B_neg_y * w_val; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 8) { + + __m512d w_x_i_vec = _mm512_loadu_pd(&w_x[i]); + __m512d w_y_i_vec = _mm512_loadu_pd(&w_y[i]); + __m512d v_max_i_vec = _mm512_loadu_pd(&v_max[i]); + __m512d pos_x_i_vec = _mm512_loadu_pd(&pos_x[i]); + __m512d pos_y_i_vec = _mm512_loadu_pd(&pos_y[i]); + __m512d destination_y_i_vec = _mm512_loadu_pd(&destination_y[i]); + __m512d destination_x_i_vec = _mm512_loadu_pd(&destination_x[i]); + __m512d desired_v0_i_vec = _mm512_loadu_pd(&desired_v0[i]); + + __m256i color_i_vec = _mm256_loadu_si256((__m256i *)&color[i]); + + __m512d w_norm_sq = _mm512_fmadd_pd(w_x_i_vec, w_x_i_vec, _mm512_mul_pd(w_y_i_vec, w_y_i_vec)); + __m512d w_norm = _mm512_sqrt_pd(w_norm_sq); + __m512d correction_factor = _mm512_min_pd(_mm512_set1_pd(1.0), _mm512_div_pd(v_max_i_vec, w_norm)); + + __m512d curr_vel_x_i_vec = _mm512_mul_pd(w_x_i_vec, correction_factor); + __m512d curr_vel_y_i_vec = _mm512_mul_pd(w_y_i_vec, correction_factor); + + __m512d dt_vec = _mm512_set1_pd(dt); + pos_x_i_vec = _mm512_fmadd_pd(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); + pos_y_i_vec = _mm512_fmadd_pd(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); + + __mmask8 cond_1_left = _mm256_cmpeq_epi32_mask(color_i_vec, _mm256_setzero_si256()); + __mmask8 cond_1_right = _mm512_cmp_pd_mask(pos_x_i_vec, _mm512_set1_pd(BOARD_LENGTH - 1.5), _CMP_GT_OQ); + __mmask8 cond_1 = cond_1_left & cond_1_right; + + pos_x_i_vec = _mm512_mask_blend_pd(cond_1, pos_x_i_vec, _mm512_set1_pd(1.0)); + destination_y_i_vec = _mm512_mask_blend_pd(cond_1, destination_y_i_vec, pos_y_i_vec); + w_x_i_vec = _mm512_mask_blend_pd(cond_1, w_x_i_vec, desired_v0_i_vec); + w_y_i_vec = _mm512_mask_blend_pd(cond_1, w_y_i_vec, _mm512_setzero_pd()); + curr_vel_x_i_vec = _mm512_mask_blend_pd(cond_1, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_pd(cond_1, curr_vel_y_i_vec, w_y_i_vec); + + __mmask8 cond_2_left = _mm256_cmpeq_epi32_mask(color_i_vec, _mm256_set1_epi32(1)); + __mmask8 cond_2_right = _mm512_cmp_pd_mask(pos_x_i_vec, _mm512_set1_pd(1.5), _CMP_LT_OQ); + __mmask8 cond_2 = cond_2_left & cond_2_right; + + pos_x_i_vec = _mm512_mask_blend_pd(cond_2, pos_x_i_vec, _mm512_set1_pd(BOARD_LENGTH - 1.0)); + destination_y_i_vec = _mm512_mask_blend_pd(cond_2, destination_y_i_vec, pos_y_i_vec); + __m512d neg_desired_v0 = _mm512_xor_pd(desired_v0_i_vec, _mm512_set1_pd(-0.0)); + w_x_i_vec = _mm512_mask_blend_pd(cond_2, w_x_i_vec, neg_desired_v0); + w_y_i_vec = _mm512_mask_blend_pd(cond_2, w_y_i_vec, _mm512_setzero_pd()); + curr_vel_x_i_vec = _mm512_mask_blend_pd(cond_2, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_pd(cond_2, curr_vel_y_i_vec, w_y_i_vec); + + pos_y_i_vec = _mm512_max_pd(pos_y_i_vec, _mm512_set1_pd(WALL_EPS)); + pos_y_i_vec = _mm512_min_pd(pos_y_i_vec, _mm512_set1_pd(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512d ex_vec = _mm512_sub_pd(destination_x_i_vec, pos_x_i_vec); + __m512d ey_vec = _mm512_sub_pd(destination_y_i_vec, pos_y_i_vec); + __m512d e_norm_sq = _mm512_fmadd_pd(ex_vec, ex_vec, _mm512_mul_pd(ey_vec, ey_vec)); + __m512d e_norm = _mm512_sqrt_pd(e_norm_sq); + __m512d e_norm_inv = _mm512_div_pd(_mm512_set1_pd(1.0), e_norm); + __m512d e_x_i_vec = _mm512_mul_pd(ex_vec, e_norm_inv); + __m512d e_y_i_vec = _mm512_mul_pd(ey_vec, e_norm_inv); + + __m512d vel_norm_sq = _mm512_fmadd_pd(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_pd(curr_vel_y_i_vec, curr_vel_y_i_vec)); + __m512d sb_sq_i_vec = _mm512_mul_pd(vel_norm_sq, _mm512_set1_pd(delta_t_sq)); + __m512d vel_norm_dt = _mm512_sqrt_pd(sb_sq_i_vec); + + _mm512_storeu_pd(&curr_vel_x[i], curr_vel_x_i_vec); + _mm512_storeu_pd(&curr_vel_y[i], curr_vel_y_i_vec); + _mm512_storeu_pd(&w_x[i], w_x_i_vec); + _mm512_storeu_pd(&w_y[i], w_y_i_vec); + _mm512_storeu_pd(&pos_x[i], pos_x_i_vec); + _mm512_storeu_pd(&pos_y[i], pos_y_i_vec); + _mm512_storeu_pd(&e_x[i], e_x_i_vec); + _mm512_storeu_pd(&e_y[i], e_y_i_vec); + _mm512_storeu_pd(&destination_y[i], destination_y_i_vec); + _mm512_storeu_pd(&sb_sq[i], sb_sq_i_vec); + _mm512_storeu_pd(&sb_e_x[i], _mm512_mul_pd(e_x_i_vec, vel_norm_dt)); + _mm512_storeu_pd(&sb_e_y[i], _mm512_mul_pd(e_y_i_vec, vel_norm_dt)); + } +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +}