From dec2fabfba70a2ab1fe51af9d1d4f8a6adb600b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=5BQuy=20Anh=5D=20=C2=ABElliot=C2=BB=20Nguyen?= <23906-nguyenquy@users.noreply.gitlab.inf.ethz.ch> Date: Thu, 28 May 2026 21:58:04 +0200 Subject: [PATCH] Added AVX-512 for float version + Vectorised second pass --- ...iant_float_inlining_SoA_AVX512_polyexp.cpp | 418 ++++++++++++++++ ...nlining_SoA_AVX512_polyexp_fastinvsqrt.cpp | 431 +++++++++++++++++ ...at_inlining_SoA_AVX512_polyexp_moreavx.cpp | 449 ++++++++++++++++++ 3 files changed, 1298 insertions(+) create mode 100644 variants/variant_float_inlining_SoA_AVX512_polyexp.cpp create mode 100644 variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp create mode 100644 variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp new file mode 100644 index 0000000..4f6d471 --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp.cpp @@ -0,0 +1,418 @@ +// # Changes: Ported AVX2 to AVX512 +// # Predecessor: variant_float_inlining_SoA_AVX2_polyexp.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 15) & ~15; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x = _mm512_setzero_ps(); + __m512 acc_y = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 16) { + __m512 j_idx = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + + __mmask16 m_neq_i = _mm512_cmp_ps_mask(j_idx, v_i, _CMP_NEQ_OQ); + __mmask16 m_in_range = _mm512_cmp_ps_mask(j_idx, v_np, _CMP_LT_OQ); + __mmask16 active = m_neq_i & m_in_range; + + __m512 pjx = _mm512_load_ps(&pos_x[j]); + __m512 pjy = _mm512_load_ps(&pos_y[j]); + __m512 sbe_jx = _mm512_load_ps(&sb_e_x[j]); + __m512 sbe_jy = _mm512_load_ps(&sb_e_y[j]); + __m512 sb_jsq = _mm512_load_ps(&sb_sq[j]); + + __m512 r_ab_x = _mm512_sub_ps(pix, pjx); + __m512 r_ab_y = _mm512_sub_ps(piy, pjy); + __m512 r_ab_n2 = _mm512_fmadd_ps(r_ab_x, r_ab_x, _mm512_mul_ps(r_ab_y, r_ab_y)); + __m512 r_ab_n = _mm512_sqrt_ps(r_ab_n2); + r_ab_n = _mm512_max_ps(r_ab_n, v_eps); + + __m512 r_ab2_x = _mm512_sub_ps(r_ab_x, sbe_jx); + __m512 r_ab2_y = _mm512_sub_ps(r_ab_y, sbe_jy); + __m512 r_ab2_n2 = _mm512_fmadd_ps(r_ab2_x, r_ab2_x, _mm512_mul_ps(r_ab2_y, r_ab2_y)); + __m512 r_ab2_n = _mm512_sqrt_ps(r_ab2_n2); + r_ab2_n = _mm512_max_ps(r_ab2_n, v_eps); + + __m512 r_sum = _mm512_add_ps(r_ab_n, r_ab2_n); + __m512 two_b_sq = _mm512_fmsub_ps(r_sum, r_sum, sb_jsq); + two_b_sq = _mm512_max_ps(two_b_sq, v_eps_sq); + __m512 two_b = _mm512_sqrt_ps(two_b_sq); + + __m512 e_arg = _mm512_mul_ps(two_b, v_neg_2s_inv); + e_arg = _mm512_max_ps(e_arg, v_exp_min); + __m512 e_val = _avx512_exp_ps(e_arg); + + __m512 numer = _mm512_mul_ps(_mm512_mul_ps(r_sum, v_neg_v0_div_2s), e_val); + __m512 denom = _mm512_mul_ps(_mm512_mul_ps(two_b, r_ab_n), r_ab2_n); + + __m512 denom_safe = _mm512_mask_blend_ps(active, v_one, denom); + __m512 smc = _mm512_div_ps(numer, denom_safe); + + __m512 sm_r_ab = _mm512_mul_ps(smc, r_ab2_n); + __m512 sm_r_ab2 = _mm512_mul_ps(smc, r_ab_n); + + __m512 f_x = _mm512_fmadd_ps(r_ab2_x, sm_r_ab2, _mm512_mul_ps(r_ab_x, sm_r_ab)); + __m512 f_y = _mm512_fmadd_ps(r_ab2_y, sm_r_ab2, _mm512_mul_ps(r_ab_y, sm_r_ab)); + + __m512 f_norm_sq = _mm512_fmadd_ps(f_x, f_x, _mm512_mul_ps(f_y, f_y)); + __m512 rhs = _mm512_mul_ps(f_norm_sq, v_cos_phi_sq); + __m512 e_dot_f = _mm512_fmadd_ps(eix, f_x, _mm512_mul_ps(eiy, f_y)); + __m512 e_dot_f_sq = _mm512_mul_ps(e_dot_f, e_dot_f); + + __mmask16 c1 = _mm512_cmp_ps_mask(e_dot_f, v_zero, _CMP_LT_OQ); + __mmask16 c2 = _mm512_cmp_ps_mask(e_dot_f_sq, rhs, _CMP_GT_OQ); + __mmask16 cond = c1 & c2; + + __m512 w_val = _mm512_mask_blend_ps(cond, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe = _mm512_maskz_mov_ps(active, f_x); + __m512 f_y_safe = _mm512_maskz_mov_ps(active, f_y); + + acc_x = _mm512_fmadd_ps(f_x_safe, w_val, acc_x); + acc_y = _mm512_fmadd_ps(f_y_safe, w_val, acc_y); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(acc_x); + f_alpha_dt_y += _mm512_reduce_add_ps(acc_y); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + float w_norm = sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + float w_norm2 = w_norm > EPS ? w_norm : EPS; + float correction_factor = min(1.0f, v_max[i] / w_norm2); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5f)) { + pos_x[i] = 1.0f; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0f; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] == 1 && pos_x[i] < 1.5f) { + pos_x[i] = BOARD_LENGTH - 1.0f; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0f; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + float ex = destination_x[i] - pos_x[i]; + float ey = destination_y[i] - pos_y[i]; + + float e_norm = sqrt(ex * ex + ey * ey); + float e_norm2 = e_norm > EPS ? e_norm : EPS; + float e_norm_inv = 1.0f / e_norm2; + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + float velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + float velocity_norm = sqrt(velocity_norm_sq); + + sb_sq[i] = velocity_norm_sq * delta_t_sq; + sb_e_x[i] = e_x[i] * (velocity_norm * delta_t); + sb_e_y[i] = e_y[i] * (velocity_norm * delta_t); + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp new file mode 100644 index 0000000..d3fcc84 --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_fastinvsqrt.cpp @@ -0,0 +1,431 @@ +// # Changes: Removed EPS checks and implemented Fast Inverse Square Root +// # Predecessor: variant_float_inlining_SoA_AVX512_polyexp.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +static inline float fast_inv_sqrt(float number) { + float y = number; + + uint32_t i = bit_cast(y); + + i = 0x5f3759df - (i >> 1); + + y = bit_cast(i); + + float x2 = number * 0.5; + y = y * (1.5 - (x2 * y * y)); + y = y * (1.5 - (x2 * y * y)); + + return y; +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 15) & ~15; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x = _mm512_setzero_ps(); + __m512 acc_y = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 16) { + __m512 j_idx = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + + __mmask16 m_neq_i = _mm512_cmp_ps_mask(j_idx, v_i, _CMP_NEQ_OQ); + __mmask16 m_in_range = _mm512_cmp_ps_mask(j_idx, v_np, _CMP_LT_OQ); + __mmask16 active = m_neq_i & m_in_range; + + __m512 pjx = _mm512_load_ps(&pos_x[j]); + __m512 pjy = _mm512_load_ps(&pos_y[j]); + __m512 sbe_jx = _mm512_load_ps(&sb_e_x[j]); + __m512 sbe_jy = _mm512_load_ps(&sb_e_y[j]); + __m512 sb_jsq = _mm512_load_ps(&sb_sq[j]); + + __m512 r_ab_x = _mm512_sub_ps(pix, pjx); + __m512 r_ab_y = _mm512_sub_ps(piy, pjy); + __m512 r_ab_n2 = _mm512_fmadd_ps(r_ab_x, r_ab_x, _mm512_mul_ps(r_ab_y, r_ab_y)); + __m512 r_ab_n = _mm512_sqrt_ps(r_ab_n2); + r_ab_n = _mm512_max_ps(r_ab_n, v_eps); + + __m512 r_ab2_x = _mm512_sub_ps(r_ab_x, sbe_jx); + __m512 r_ab2_y = _mm512_sub_ps(r_ab_y, sbe_jy); + __m512 r_ab2_n2 = _mm512_fmadd_ps(r_ab2_x, r_ab2_x, _mm512_mul_ps(r_ab2_y, r_ab2_y)); + __m512 r_ab2_n = _mm512_sqrt_ps(r_ab2_n2); + r_ab2_n = _mm512_max_ps(r_ab2_n, v_eps); + + __m512 r_sum = _mm512_add_ps(r_ab_n, r_ab2_n); + __m512 two_b_sq = _mm512_fmsub_ps(r_sum, r_sum, sb_jsq); + two_b_sq = _mm512_max_ps(two_b_sq, v_eps_sq); + __m512 two_b = _mm512_sqrt_ps(two_b_sq); + + __m512 e_arg = _mm512_mul_ps(two_b, v_neg_2s_inv); + e_arg = _mm512_max_ps(e_arg, v_exp_min); + __m512 e_val = _avx512_exp_ps(e_arg); + + __m512 numer = _mm512_mul_ps(_mm512_mul_ps(r_sum, v_neg_v0_div_2s), e_val); + __m512 denom = _mm512_mul_ps(_mm512_mul_ps(two_b, r_ab_n), r_ab2_n); + + __m512 denom_safe = _mm512_mask_blend_ps(active, v_one, denom); + __m512 smc = _mm512_div_ps(numer, denom_safe); + + __m512 sm_r_ab = _mm512_mul_ps(smc, r_ab2_n); + __m512 sm_r_ab2 = _mm512_mul_ps(smc, r_ab_n); + + __m512 f_x = _mm512_fmadd_ps(r_ab2_x, sm_r_ab2, _mm512_mul_ps(r_ab_x, sm_r_ab)); + __m512 f_y = _mm512_fmadd_ps(r_ab2_y, sm_r_ab2, _mm512_mul_ps(r_ab_y, sm_r_ab)); + + __m512 f_norm_sq = _mm512_fmadd_ps(f_x, f_x, _mm512_mul_ps(f_y, f_y)); + __m512 rhs = _mm512_mul_ps(f_norm_sq, v_cos_phi_sq); + __m512 e_dot_f = _mm512_fmadd_ps(eix, f_x, _mm512_mul_ps(eiy, f_y)); + __m512 e_dot_f_sq = _mm512_mul_ps(e_dot_f, e_dot_f); + + __mmask16 c1 = _mm512_cmp_ps_mask(e_dot_f, v_zero, _CMP_LT_OQ); + __mmask16 c2 = _mm512_cmp_ps_mask(e_dot_f_sq, rhs, _CMP_GT_OQ); + __mmask16 cond = c1 & c2; + + __m512 w_val = _mm512_mask_blend_ps(cond, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe = _mm512_maskz_mov_ps(active, f_x); + __m512 f_y_safe = _mm512_maskz_mov_ps(active, f_y); + + acc_x = _mm512_fmadd_ps(f_x_safe, w_val, acc_x); + acc_y = _mm512_fmadd_ps(f_y_safe, w_val, acc_y); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(acc_x); + f_alpha_dt_y += _mm512_reduce_add_ps(acc_y); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + float w_norm_inv = fast_inv_sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + float correction_factor = min(1.0f, v_max[i] * w_norm_inv); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5f)) { + pos_x[i] = 1.0f; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0f; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] == 1 && pos_x[i] < 1.5f) { + pos_x[i] = BOARD_LENGTH - 1.0f; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0f; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + float ex = destination_x[i] - pos_x[i]; + float ey = destination_y[i] - pos_y[i]; + + float e_norm_inv = fast_inv_sqrt(ex * ex + ey * ey); + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + double velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + double sb_sq_i = velocity_norm_sq * delta_t_sq; + double velocity_norm_mul_delta_t = sqrt(sb_sq_i); + + sb_sq[i] = sb_sq_i; + sb_e_x[i] = e_x[i] * velocity_norm_mul_delta_t; + sb_e_y[i] = e_y[i] * velocity_norm_mul_delta_t; + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp new file mode 100644 index 0000000..28cb9b2 --- /dev/null +++ b/variants/variant_float_inlining_SoA_AVX512_polyexp_moreavx.cpp @@ -0,0 +1,449 @@ +// # Changes: Vectorised second pass with AVX-512 +// # Predecessor: variant_float_inlining_SoA_AVX512_polyexp.cpp + +#include "../headers/board_reduced_float.hpp" +#include "../headers/constants_float.hpp" +#include "../headers/pedestrian_reduced_float.hpp" + +#include + +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0f +#define BOARD_LENGTH 50.0f +#define WALL_EPS 1e-3f +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999f + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 15) & ~15; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + float x = x_gen_left(gen); + float y = y_gen(gen); + + float dest_x = BOARD_LENGTH + 30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int i = n_p_half; i < n_p; i++) { + float x = x_gen_right(gen); + float y = y_gen(gen); + + float dest_x = -30.0f; + float dest_y = y; + + float init_e_x = dest_x - x; + float init_e_y = dest_y - y; + float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + float init_e_unit_x = init_e_x / init_e_norm; + float init_e_unit_y = init_e_y / init_e_norm; + + float d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1f); + + float v_m = 1.3f * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0f; + w_y[i] = 0.0f; + curr_vel_x[i] = 0.0f; + curr_vel_y[i] = 0.0f; + sb_sq[i] = 0.0f; + sb_e_x[i] = 0.0f; + sb_e_y[i] = 0.0f; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9f; + pos_y[j] = 1e9f; + sb_sq[j] = 0.0f; + sb_e_x[j] = 0.0f; + sb_e_y[j] = 0.0f; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512 _avx512_exp_ps(__m512 x) { + const __m512 log2e = _mm512_set1_ps(1.4426950408889634f); + __m512 nf = _mm512_roundscale_ps(_mm512_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + + const __m512 ln2 = _mm512_set1_ps(0.6931471805599453f); + __m512 r = _mm512_fnmadd_ps(nf, ln2, x); + + const __m512 c5 = _mm512_set1_ps(1.0f / 120.0f); + const __m512 c4 = _mm512_set1_ps(1.0f / 24.0f); + const __m512 c3 = _mm512_set1_ps(1.0f / 6.0f); + const __m512 c2 = _mm512_set1_ps(0.5f); + const __m512 c1 = _mm512_set1_ps(1.0f); + const __m512 c0 = _mm512_set1_ps(1.0f); + + __m512 p = c5; + p = _mm512_fmadd_ps(p, r, c4); + p = _mm512_fmadd_ps(p, r, c3); + p = _mm512_fmadd_ps(p, r, c2); + p = _mm512_fmadd_ps(p, r, c1); + p = _mm512_fmadd_ps(p, r, c0); + + __m512i ni = _mm512_cvtps_epi32(nf); + __m512i biased = _mm512_add_epi32(ni, _mm512_set1_epi32(127)); + __m512i scaled = _mm512_slli_epi32(biased, 23); + __m512 pow2n = _mm512_castsi512_ps(scaled); + + return _mm512_mul_ps(p, pow2n); +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + (void)pedestrians; +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 15) & ~15; + + alignas(64) const float lane_offset_arr[16] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}; + const __m512 lane_off = _mm512_load_ps(lane_offset_arr); + + const __m512 v_neg_v0_div_2s = _mm512_set1_ps(NEG_V0_AB_div_TWO_SIGMA); + const __m512 v_neg_2s_inv = _mm512_set1_ps(NEG_TWO_SIGMA_INV); + const __m512 v_cos_phi_sq = _mm512_set1_ps(cos_phi_sq); + const __m512 v_dt_neg = _mm512_set1_ps(dt_neg); + const __m512 v_c_neg_dt = _mm512_set1_ps(c_neg_times_dt); + const __m512 v_zero = _mm512_setzero_ps(); + const __m512 v_one = _mm512_set1_ps(1.0f); + const __m512 v_eps = _mm512_set1_ps(EPS); + const __m512 v_eps_sq = _mm512_set1_ps(EPS_SQ); + const __m512 v_np = _mm512_set1_ps((float)n_p); + const __m512 v_exp_min = _mm512_set1_ps(-87.0f); + + for (int t = 0; t < n_iter; t++) { + for (int i = 0; i < n_p; i++) { + float desired_velocity_x = e_x[i] * desired_v0[i]; + float desired_velocity_y = e_y[i] * desired_v0[i]; + + float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; + float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; + + const __m512 pix = _mm512_set1_ps(pos_x[i]); + const __m512 piy = _mm512_set1_ps(pos_y[i]); + const __m512 eix = _mm512_set1_ps(e_x[i]); + const __m512 eiy = _mm512_set1_ps(e_y[i]); + const __m512 v_i = _mm512_set1_ps((float)i); + + __m512 acc_x = _mm512_setzero_ps(); + __m512 acc_y = _mm512_setzero_ps(); + + for (int j = 0; j < n_p_pad; j += 16) { + __m512 j_idx = _mm512_add_ps(_mm512_set1_ps((float)j), lane_off); + + __mmask16 m_neq_i = _mm512_cmp_ps_mask(j_idx, v_i, _CMP_NEQ_OQ); + __mmask16 m_in_range = _mm512_cmp_ps_mask(j_idx, v_np, _CMP_LT_OQ); + __mmask16 active = m_neq_i & m_in_range; + + __m512 pjx = _mm512_load_ps(&pos_x[j]); + __m512 pjy = _mm512_load_ps(&pos_y[j]); + __m512 sbe_jx = _mm512_load_ps(&sb_e_x[j]); + __m512 sbe_jy = _mm512_load_ps(&sb_e_y[j]); + __m512 sb_jsq = _mm512_load_ps(&sb_sq[j]); + + __m512 r_ab_x = _mm512_sub_ps(pix, pjx); + __m512 r_ab_y = _mm512_sub_ps(piy, pjy); + __m512 r_ab_n2 = _mm512_fmadd_ps(r_ab_x, r_ab_x, _mm512_mul_ps(r_ab_y, r_ab_y)); + __m512 r_ab_n = _mm512_sqrt_ps(r_ab_n2); + r_ab_n = _mm512_max_ps(r_ab_n, v_eps); + + __m512 r_ab2_x = _mm512_sub_ps(r_ab_x, sbe_jx); + __m512 r_ab2_y = _mm512_sub_ps(r_ab_y, sbe_jy); + __m512 r_ab2_n2 = _mm512_fmadd_ps(r_ab2_x, r_ab2_x, _mm512_mul_ps(r_ab2_y, r_ab2_y)); + __m512 r_ab2_n = _mm512_sqrt_ps(r_ab2_n2); + r_ab2_n = _mm512_max_ps(r_ab2_n, v_eps); + + __m512 r_sum = _mm512_add_ps(r_ab_n, r_ab2_n); + __m512 two_b_sq = _mm512_fmsub_ps(r_sum, r_sum, sb_jsq); + two_b_sq = _mm512_max_ps(two_b_sq, v_eps_sq); + __m512 two_b = _mm512_sqrt_ps(two_b_sq); + + __m512 e_arg = _mm512_mul_ps(two_b, v_neg_2s_inv); + e_arg = _mm512_max_ps(e_arg, v_exp_min); + __m512 e_val = _avx512_exp_ps(e_arg); + + __m512 numer = _mm512_mul_ps(_mm512_mul_ps(r_sum, v_neg_v0_div_2s), e_val); + __m512 denom = _mm512_mul_ps(_mm512_mul_ps(two_b, r_ab_n), r_ab2_n); + + __m512 denom_safe = _mm512_mask_blend_ps(active, v_one, denom); + __m512 smc = _mm512_div_ps(numer, denom_safe); + + __m512 sm_r_ab = _mm512_mul_ps(smc, r_ab2_n); + __m512 sm_r_ab2 = _mm512_mul_ps(smc, r_ab_n); + + __m512 f_x = _mm512_fmadd_ps(r_ab2_x, sm_r_ab2, _mm512_mul_ps(r_ab_x, sm_r_ab)); + __m512 f_y = _mm512_fmadd_ps(r_ab2_y, sm_r_ab2, _mm512_mul_ps(r_ab_y, sm_r_ab)); + + __m512 f_norm_sq = _mm512_fmadd_ps(f_x, f_x, _mm512_mul_ps(f_y, f_y)); + __m512 rhs = _mm512_mul_ps(f_norm_sq, v_cos_phi_sq); + __m512 e_dot_f = _mm512_fmadd_ps(eix, f_x, _mm512_mul_ps(eiy, f_y)); + __m512 e_dot_f_sq = _mm512_mul_ps(e_dot_f, e_dot_f); + + __mmask16 c1 = _mm512_cmp_ps_mask(e_dot_f, v_zero, _CMP_LT_OQ); + __mmask16 c2 = _mm512_cmp_ps_mask(e_dot_f_sq, rhs, _CMP_GT_OQ); + __mmask16 cond = c1 & c2; + + __m512 w_val = _mm512_mask_blend_ps(cond, v_dt_neg, v_c_neg_dt); + + __m512 f_x_safe = _mm512_maskz_mov_ps(active, f_x); + __m512 f_y_safe = _mm512_maskz_mov_ps(active, f_y); + + acc_x = _mm512_fmadd_ps(f_x_safe, w_val, acc_x); + acc_y = _mm512_fmadd_ps(f_y_safe, w_val, acc_y); + } + + f_alpha_dt_x += _mm512_reduce_add_ps(acc_x); + f_alpha_dt_y += _mm512_reduce_add_ps(acc_y); + + for (int k = 0; k < board.n_obstacles; k++) { + float v1_x = pos_x[i] - board.obstacles[k].start_x; + float v1_y = pos_y[i] - board.obstacles[k].start_y; + float v2_x = board.obstacles[k].obstacle_v_x; + float v2_y = board.obstacles[k].obstacle_v_y; + + float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0f) + projection = 0.0f; + else if (projection > 1.0f) + projection = 1.0f; + + float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; + + float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + float f_alpha_B_neg_x = r_aB_x * scalar_mult; + float f_alpha_B_neg_y = r_aB_y * scalar_mult; + + float w_val = dt_neg; + float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); + + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; + float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; + + f_alpha_dt_x += f_alpha_B_dt_x; + f_alpha_dt_y += f_alpha_B_dt_y; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p_pad; i += 16) { + + __m512 w_x_i_vec = _mm512_loadu_ps(&w_x[i]); + __m512 w_y_i_vec = _mm512_loadu_ps(&w_y[i]); + __m512 v_max_i_vec = _mm512_loadu_ps(&v_max[i]); + __m512 pos_x_i_vec = _mm512_loadu_ps(&pos_x[i]); + __m512 pos_y_i_vec = _mm512_loadu_ps(&pos_y[i]); + __m512 destination_y_i_vec = _mm512_loadu_ps(&destination_y[i]); + __m512 destination_x_i_vec = _mm512_loadu_ps(&destination_x[i]); + __m512 desired_v0_i_vec = _mm512_loadu_ps(&desired_v0[i]); + + __m512i color_i_vec = _mm512_loadu_si512((__m512i*)&color[i]); + + __m512 w_norm_sq = _mm512_fmadd_ps(w_x_i_vec, w_x_i_vec, _mm512_mul_ps(w_y_i_vec, w_y_i_vec)); + __m512 w_norm = _mm512_sqrt_ps(w_norm_sq); + __m512 w_norm_safe = _mm512_max_ps(w_norm, _mm512_set1_ps(EPS)); + __m512 correction_factor = _mm512_min_ps(_mm512_set1_ps(1.0f), _mm512_div_ps(v_max_i_vec, w_norm_safe)); + + __m512 curr_vel_x_i_vec = _mm512_mul_ps(w_x_i_vec, correction_factor); + __m512 curr_vel_y_i_vec = _mm512_mul_ps(w_y_i_vec, correction_factor); + + __m512 dt_vec = _mm512_set1_ps(dt); + pos_x_i_vec = _mm512_fmadd_ps(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); + pos_y_i_vec = _mm512_fmadd_ps(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); + + __mmask16 cond_1_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_setzero_si512()); + __mmask16 cond_1_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.5f), _CMP_GT_OQ); + __mmask16 cond_1 = cond_1_left & cond_1_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_1, pos_x_i_vec, _mm512_set1_ps(1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_1, destination_y_i_vec, pos_y_i_vec); + w_x_i_vec = _mm512_mask_blend_ps(cond_1, w_x_i_vec, desired_v0_i_vec); + w_y_i_vec = _mm512_mask_blend_ps(cond_1, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_1, curr_vel_y_i_vec, w_y_i_vec); + + __mmask16 cond_2_left = _mm512_cmpeq_epi32_mask(color_i_vec, _mm512_set1_epi32(1)); + __mmask16 cond_2_right = _mm512_cmp_ps_mask(pos_x_i_vec, _mm512_set1_ps(1.5f), _CMP_LT_OQ); + __mmask16 cond_2 = cond_2_left & cond_2_right; + + pos_x_i_vec = _mm512_mask_blend_ps(cond_2, pos_x_i_vec, _mm512_set1_ps(BOARD_LENGTH - 1.0f)); + destination_y_i_vec = _mm512_mask_blend_ps(cond_2, destination_y_i_vec, pos_y_i_vec); + __m512 neg_desired_v0 = _mm512_xor_ps(desired_v0_i_vec, _mm512_set1_ps(-0.0f)); + w_x_i_vec = _mm512_mask_blend_ps(cond_2, w_x_i_vec, neg_desired_v0); + w_y_i_vec = _mm512_mask_blend_ps(cond_2, w_y_i_vec, _mm512_setzero_ps()); + curr_vel_x_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_x_i_vec, w_x_i_vec); + curr_vel_y_i_vec = _mm512_mask_blend_ps(cond_2, curr_vel_y_i_vec, w_y_i_vec); + + pos_y_i_vec = _mm512_max_ps(pos_y_i_vec, _mm512_set1_ps(WALL_EPS)); + pos_y_i_vec = _mm512_min_ps(pos_y_i_vec, _mm512_set1_ps(BOARD_WIDTH_MINUS_WALL_EPS)); + + __m512 ex_vec = _mm512_sub_ps(destination_x_i_vec, pos_x_i_vec); + __m512 ey_vec = _mm512_sub_ps(destination_y_i_vec, pos_y_i_vec); + __m512 e_norm_sq = _mm512_fmadd_ps(ex_vec, ex_vec, _mm512_mul_ps(ey_vec, ey_vec)); + __m512 e_norm = _mm512_sqrt_ps(e_norm_sq); + __m512 e_norm_inv = _mm512_div_ps(_mm512_set1_ps(1.0f), e_norm); + __m512 e_x_i_vec = _mm512_mul_ps(ex_vec, e_norm_inv); + __m512 e_y_i_vec = _mm512_mul_ps(ey_vec, e_norm_inv); + + __m512 vel_norm_sq = _mm512_fmadd_ps(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_ps(curr_vel_y_i_vec, curr_vel_y_i_vec)); + __m512 sb_sq_i_vec = _mm512_mul_ps(vel_norm_sq, _mm512_set1_ps(delta_t_sq)); + __m512 vel_norm_dt = _mm512_sqrt_ps(sb_sq_i_vec); + + _mm512_storeu_ps(&curr_vel_x[i], curr_vel_x_i_vec); + _mm512_storeu_ps(&curr_vel_y[i], curr_vel_y_i_vec); + _mm512_storeu_ps(&w_x[i], w_x_i_vec); + _mm512_storeu_ps(&w_y[i], w_y_i_vec); + _mm512_storeu_ps(&pos_x[i], pos_x_i_vec); + _mm512_storeu_ps(&pos_y[i], pos_y_i_vec); + _mm512_storeu_ps(&e_x[i], e_x_i_vec); + _mm512_storeu_ps(&e_y[i], e_y_i_vec); + _mm512_storeu_ps(&destination_y[i], destination_y_i_vec); + _mm512_storeu_ps(&sb_sq[i], sb_sq_i_vec); + _mm512_storeu_ps(&sb_e_x[i], _mm512_mul_ps(e_x_i_vec, vel_norm_dt)); + _mm512_storeu_ps(&sb_e_y[i], _mm512_mul_ps(e_y_i_vec, vel_norm_dt)); + } + +#if !BENCHMARK + for (int i = 0; i < n_p; i++) { + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +}