From 7448281515e07760106addd34d9319f0b5d55399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=5BQuy=20Anh=5D=20=C2=ABElliot=C2=BB=20Nguyen?= <23906-nguyenquy@users.noreply.gitlab.inf.ethz.ch> Date: Mon, 25 May 2026 14:01:10 +0200 Subject: [PATCH] variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp --- Makefile | 2 +- headers/forces.hpp | 6 +- tests/test_forces.cpp | 18 +- ...ining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp | 447 ++++++++++++++++ ...ng_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp | 476 ++++++++++++++++++ 5 files changed, 938 insertions(+), 11 deletions(-) create mode 100644 variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp create mode 100644 variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp diff --git a/Makefile b/Makefile index 37b2e6e..af08a59 100644 --- a/Makefile +++ b/Makefile @@ -201,4 +201,4 @@ clean: # WARNING: this deletes all *.csv and *.mp4 files in this directory. # Move any output files you want to keep to another directory before running it. clean-all: clean - rm -f *.csv *.mp4 + rm -f *.csv *.mp4 \ No newline at end of file diff --git a/headers/forces.hpp b/headers/forces.hpp index fe2453a..8b6436a 100644 --- a/headers/forces.hpp +++ b/headers/forces.hpp @@ -37,7 +37,11 @@ // @param f : The force under consideration (e.g., the repulsive force from another pedestrian or an obstacle). // @return The direction-dependent perception weight multiplied by -dt. // -double w(Vector2d e, Vector2d f); +// pedestrian-pedestrian weighting (full e dot f vs ||f||*cos(phi) check) +double w_ped(Vector2d e, Vector2d f); + +// obstacle weighting (cheaper: obstacles are x-axis walls so only f.y matters) +double w_obstacle(Vector2d e, Vector2d f); // Computes the acceleration term of the social force model // for pedestrian alpha (Eq. (2) in the paper): diff --git a/tests/test_forces.cpp b/tests/test_forces.cpp index 38da0ed..4a19b29 100644 --- a/tests/test_forces.cpp +++ b/tests/test_forces.cpp @@ -37,36 +37,36 @@ StepResult IntegrateOneStep(const Pedestrian &p, Vector2d total_force_dt) { } // namespace // ============================================================================= -// Tests for w(e, f) +// Tests for w_ped(e, f) // ============================================================================= -TEST(WeightFn, ForceAlignedWithDirectionIsInView) { EXPECT_DOUBLE_EQ(w(Vector2d(1.0, 0.0), Vector2d(1.0, 0.0)), dt_neg); } +TEST(WeightFn, ForceAlignedWithDirectionIsInView) { EXPECT_DOUBLE_EQ(w_ped(Vector2d(1.0, 0.0), Vector2d(1.0, 0.0)), dt_neg); } -TEST(WeightFn, ForceOppositeToDirectionIsOutOfView) { EXPECT_DOUBLE_EQ(w(Vector2d(1.0, 0.0), Vector2d(-1.0, 0.0)), c_neg_times_dt); } +TEST(WeightFn, ForceOppositeToDirectionIsOutOfView) { EXPECT_DOUBLE_EQ(w_ped(Vector2d(1.0, 0.0), Vector2d(-1.0, 0.0)), c_neg_times_dt); } -TEST(WeightFn, PerpendicularForceIsInView) { EXPECT_DOUBLE_EQ(w(Vector2d(1.0, 1.0), Vector2d(-1.0, 1.0)), dt_neg); } +TEST(WeightFn, PerpendicularForceIsInView) { EXPECT_DOUBLE_EQ(w_ped(Vector2d(1.0, 1.0), Vector2d(-1.0, 1.0)), dt_neg); } -TEST(WeightFn, ZeroForceIsInView) { EXPECT_DOUBLE_EQ(w(Vector2d(1.0, 0.0), Vector2d(0.0, 0.0)), dt_neg); } +TEST(WeightFn, ZeroForceIsInView) { EXPECT_DOUBLE_EQ(w_ped(Vector2d(1.0, 0.0), Vector2d(0.0, 0.0)), dt_neg); } TEST(WeightFn, AngleSlightlyInsidePhiIsInView) { Vector2d e(1.0, 0.0); const double angle = 99.0 / 180.0 * M_PI; Vector2d f(std::cos(angle), std::sin(angle)); - EXPECT_DOUBLE_EQ(w(e, f), dt_neg); + EXPECT_DOUBLE_EQ(w_ped(e, f), dt_neg); } TEST(WeightFn, AngleSlightlyOutsidePhiIsOutOfView) { Vector2d e(1.0, 0.0); const double angle = 101.0 / 180.0 * M_PI; Vector2d f(std::cos(angle), std::sin(angle)); - EXPECT_DOUBLE_EQ(w(e, f), c_neg_times_dt); + EXPECT_DOUBLE_EQ(w_ped(e, f), c_neg_times_dt); } TEST(WeightFn, ScalingForceMagnitudeDoesNotChangeView) { Vector2d e(1.0, 0.0); Vector2d f(0.7, 0.7); - EXPECT_DOUBLE_EQ(w(e, f), w(e, f * 1000.0)); - EXPECT_DOUBLE_EQ(w(e, f), w(e, f * 0.001)); + EXPECT_DOUBLE_EQ(w_ped(e, f), w_ped(e, f * 1000.0)); + EXPECT_DOUBLE_EQ(w_ped(e, f), w_ped(e, f * 0.001)); } // ============================================================================= diff --git a/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp new file mode 100644 index 0000000..758896e --- /dev/null +++ b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp @@ -0,0 +1,447 @@ +// # Changes: +// # - Ported AVX2 to AVX-512 +// # - Replaced Obstacle with the version that contains no Vector2d +// # - 64-byte aligned the vectors + replaced loadu with load +// # - Implemented Fast Inverse Square Root +// # Predecessor: variant_inlining_7_SoA_AVX2_polyexp.cpp + +#include "../headers/board_reduced.hpp" +#include "../headers/constants.hpp" +#include "../headers/pedestrian_reduced.hpp" + +#include + +#include +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0 +#define BOARD_LENGTH 50.0 +#define WALL_EPS 1e-3 +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999 + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { + +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 7) & ~7; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + double x = x_gen_left(gen); + double y = y_gen(gen); + + double dest_x = BOARD_LENGTH + 30.0; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int i = n_p_half; i < n_p; i++) { + double x = x_gen_right(gen); + double y = y_gen(gen); + + double dest_x = -30; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9; + pos_y[j] = 1e9; + sb_sq[j] = 0.0; + sb_e_x[j] = 0.0; + sb_e_y[j] = 0.0; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512d _avx512_exp_pd(__m512d x) { + + const __m512d log2e = _mm512_set1_pd(1.4426950408889634); + __m512d nf = _mm512_roundscale_pd(_mm512_mul_pd(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + const __m512d ln2 = _mm512_set1_pd(0.6931471805599453); + __m512d r = _mm512_fnmadd_pd(nf, ln2, x); + + const __m512d c7 = _mm512_set1_pd(1.0 / 5040.0); + const __m512d c6 = _mm512_set1_pd(1.0 / 720.0); + const __m512d c5 = _mm512_set1_pd(1.0 / 120.0); + const __m512d c4 = _mm512_set1_pd(1.0 / 24.0); + const __m512d c3 = _mm512_set1_pd(1.0 / 6.0); + const __m512d c2 = _mm512_set1_pd(0.5); + const __m512d c1 = _mm512_set1_pd(1.0); + const __m512d c0 = _mm512_set1_pd(1.0); + + __m512d p = c7; + p = _mm512_fmadd_pd(p, r, c6); + p = _mm512_fmadd_pd(p, r, c5); + p = _mm512_fmadd_pd(p, r, c4); + p = _mm512_fmadd_pd(p, r, c3); + p = _mm512_fmadd_pd(p, r, c2); + p = _mm512_fmadd_pd(p, r, c1); + p = _mm512_fmadd_pd(p, r, c0); + + __m256d nf_lo = _mm512_castpd512_pd256(nf); + __m256d nf_hi = _mm512_extractf64x4_pd(nf, 1); + + __m128i ni32_lo = _mm256_cvtpd_epi32(nf_lo); + __m128i ni32_hi = _mm256_cvtpd_epi32(nf_hi); + + __m256i ni64_lo = _mm256_cvtepi32_epi64(ni32_lo); + __m256i ni64_hi = _mm256_cvtepi32_epi64(ni32_hi); + + __m512i ni64 = _mm512_inserti64x4(_mm512_castsi256_si512(ni64_lo), ni64_hi, 1); + + __m512i biased = _mm512_add_epi64(ni64, _mm512_set1_epi64(1023)); + + __m512i scaled = _mm512_slli_epi64(biased, 52); + + __m512d pow2n = _mm512_castsi512_pd(scaled); + + return _mm512_mul_pd(p, pow2n); +} + +static inline double fast_inv_sqrt(double number) { + double y = number; + + uint64_t i = bit_cast(y); + + i = 0x5fe6eb50c7b537a9ULL - (i >> 1); + + y = bit_cast(i); + + double x2 = number * 0.5; + y = y * (1.5 - (x2 * y * y)); + y = y * (1.5 - (x2 * y * y)); + y = y * (1.5 - (x2 * y * y)); + + return y; +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 7) & ~7; + + alignas(64) constexpr double lane_offset_arr[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; + const __m512d lane_off = _mm512_load_pd(lane_offset_arr); + const __m512d v_neg_v0_div_2s = _mm512_set1_pd(NEG_V0_AB_div_TWO_SIGMA); + const __m512d v_neg_2s_inv = _mm512_set1_pd(NEG_TWO_SIGMA_INV); + const __m512d v_cos_phi_inv = _mm512_set1_pd(cos_phi_inv); + const __m512d v_dt_neg = _mm512_set1_pd(dt_neg); + const __m512d v_c_neg_dt = _mm512_set1_pd(c_neg_times_dt); + const __m512d v_zero = _mm512_setzero_pd(); + + const __m512d v_np = _mm512_set1_pd(n_p); + + for (int t = 0; t < n_iter; t++) { + + for (int i = 0; i < n_p; i++) { + double pos_x_i = pos_x[i]; + double pos_y_i = pos_y[i]; + double e_x_i = e_x[i]; + double e_y_i = e_y[i]; + double desired_v0_i = desired_v0[i]; + double curr_vel_x_i = curr_vel_x[i]; + double curr_vel_y_i = curr_vel_y[i]; + + double f_alpha_dt_x = (e_x_i * desired_v0_i - curr_vel_x_i) * dt_times_tau_alpha_inv; + double f_alpha_dt_y = (e_y_i * desired_v0_i - curr_vel_y_i) * dt_times_tau_alpha_inv; + + const __m512d pix = _mm512_set1_pd(pos_x_i); + const __m512d piy = _mm512_set1_pd(pos_y_i); + const __m512d eix = _mm512_set1_pd(e_x_i); + const __m512d eiy = _mm512_set1_pd(e_y_i); + const __m512d v_i = _mm512_set1_pd((double)i); + + __m512d acc_x = _mm512_setzero_pd(); + __m512d acc_y = _mm512_setzero_pd(); + + for (int j = 0; j < n_p_pad; j += 8) { + __m512d j_idx = _mm512_add_pd(_mm512_set1_pd((double)j), lane_off); + + __mmask8 m_neq_i = _mm512_cmp_pd_mask(j_idx, v_i, _CMP_NEQ_OQ); + __mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ); + __mmask8 active = m_neq_i & m_in_range; + + __m512d pjx = _mm512_load_pd(&pos_x[j]); + __m512d pjy = _mm512_load_pd(&pos_y[j]); + __m512d sbe_jx = _mm512_load_pd(&sb_e_x[j]); + __m512d sbe_jy = _mm512_load_pd(&sb_e_y[j]); + __m512d sb_jsq = _mm512_load_pd(&sb_sq[j]); + + __m512d r_ab_x = _mm512_sub_pd(pix, pjx); + __m512d r_ab_y = _mm512_sub_pd(piy, pjy); + __m512d r_ab_n2 = _mm512_fmadd_pd(r_ab_x, r_ab_x, _mm512_mul_pd(r_ab_y, r_ab_y)); + __m512d r_ab_n = _mm512_sqrt_pd(r_ab_n2); + + __m512d r_ab2_x = _mm512_sub_pd(r_ab_x, sbe_jx); + __m512d r_ab2_y = _mm512_sub_pd(r_ab_y, sbe_jy); + __m512d r_ab2_n2 = _mm512_fmadd_pd(r_ab2_x, r_ab2_x, _mm512_mul_pd(r_ab2_y, r_ab2_y)); + __m512d r_ab2_n = _mm512_sqrt_pd(r_ab2_n2); + + __m512d r_sum = _mm512_add_pd(r_ab_n, r_ab2_n); + __m512d two_b_sq = _mm512_fmsub_pd(r_sum, r_sum, sb_jsq); + __m512d two_b = _mm512_sqrt_pd(two_b_sq); + + __m512d e_arg = _mm512_mul_pd(two_b, v_neg_2s_inv); + __m512d e_val = _avx512_exp_pd(e_arg); + + __m512d numer = _mm512_mul_pd(_mm512_mul_pd(r_sum, v_neg_v0_div_2s), e_val); + __m512d denom = _mm512_mul_pd(_mm512_mul_pd(two_b, r_ab_n), r_ab2_n); + __m512d denom_safe = _mm512_mask_blend_pd(active, _mm512_set1_pd(1.0), denom); + __m512d smc = _mm512_div_pd(numer, denom_safe); + + __m512d sm_r_ab = _mm512_mul_pd(smc, r_ab2_n); + __m512d sm_r_ab2 = _mm512_mul_pd(smc, r_ab_n); + + __m512d f_x = _mm512_fmadd_pd(r_ab2_x, sm_r_ab2, _mm512_mul_pd(r_ab_x, sm_r_ab)); + __m512d f_y = _mm512_fmadd_pd(r_ab2_y, sm_r_ab2, _mm512_mul_pd(r_ab_y, sm_r_ab)); + + __m512d a = _mm512_mul_pd(_mm512_fmadd_pd(eix, f_x, _mm512_mul_pd(eiy, f_y)), v_cos_phi_inv); + __m512d a_sq = _mm512_mul_pd(a, a); + __m512d b_sq = _mm512_fmadd_pd(f_x, f_x, _mm512_mul_pd(f_y, f_y)); + + __mmask8 k_ge = _mm512_cmp_pd_mask(a, v_zero, _CMP_GE_OQ); + __mmask8 k_gt = _mm512_cmp_pd_mask(a_sq, b_sq, _CMP_GT_OQ); + __mmask8 k_cond = _mm512_kand(k_ge, k_gt); + __m512d w_val = _mm512_mask_blend_pd(k_cond, v_dt_neg, v_c_neg_dt); + + __m512d f_x_safe = _mm512_maskz_mov_pd(active, f_x); + __m512d f_y_safe = _mm512_maskz_mov_pd(active, f_y); + acc_x = _mm512_fmadd_pd(f_x_safe, w_val, acc_x); + acc_y = _mm512_fmadd_pd(f_y_safe, w_val, acc_y); + } + + f_alpha_dt_x += _mm512_reduce_add_pd(acc_x); + f_alpha_dt_y += _mm512_reduce_add_pd(acc_y); + + for (int k = 0; k < board.n_obstacles; k++) { + + double v1_x = pos_x_i - board.obstacles[k].start_x; + double v1_y = pos_y_i - board.obstacles[k].start_y; + + double projection = (v1_x * board.obstacles[k].obstacle_v_x + v1_y * board.obstacles[k].obstacle_v_y) * board.obstacles[k].length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + double r_aB_x = pos_x_i - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); + double r_aB_y = pos_y_i - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); + + double r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + double scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + double f_alpha_B_neg_x = r_aB_x * scalar_mult; + double f_alpha_B_neg_y = r_aB_y * scalar_mult; + + double w_val = dt_neg; + double e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y_i : e_y_i); + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_B_neg_x * w_val; + f_alpha_dt_y += f_alpha_B_neg_y * w_val; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + double w_norm_inv = fast_inv_sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); + + double correction_factor = min(1.0, v_max[i] * w_norm_inv); + + curr_vel_x[i] = w_x[i] * correction_factor; + curr_vel_y[i] = w_y[i] * correction_factor; + + pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; + pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; + + if (!color[i] && pos_x[i] > (BOARD_LENGTH - 1.5)) { + pos_x[i] = 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + if (color[i] && pos_x[i] < 1.5) { + pos_x[i] = BOARD_LENGTH - 1.0; + destination_y[i] = pos_y[i]; + w_x[i] = -desired_v0[i]; + w_y[i] = 0.0; + curr_vel_x[i] = w_x[i]; + curr_vel_y[i] = w_y[i]; + } + + pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + double ex = destination_x[i] - pos_x[i]; + double ey = destination_y[i] - pos_y[i]; + + double e_norm_inv = fast_inv_sqrt(ex * ex + ey * ey); + + e_x[i] = ex * e_norm_inv; + e_y[i] = ey * e_norm_inv; + + double velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; + double sb_sq_i = velocity_norm_sq * delta_t_sq; + double velocity_norm_mul_delta_t = sqrt(sb_sq_i); + + sb_sq[i] = sb_sq_i; + + sb_e_x[i] = e_x[i] * velocity_norm_mul_delta_t; + sb_e_y[i] = e_y[i] * velocity_norm_mul_delta_t; + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +} diff --git a/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp new file mode 100644 index 0000000..0651a99 --- /dev/null +++ b/variants/variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp @@ -0,0 +1,476 @@ +// # Changes: Performed Scalar Replacements +// # Predecessor: variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp + +#include "../headers/board_reduced.hpp" +#include "../headers/constants.hpp" +#include "../headers/pedestrian_reduced.hpp" + +#include + +#include +#include +#include + +#define SEED 42 +#define BOARD_WIDTH 10.0 +#define BOARD_LENGTH 50.0 +#define WALL_EPS 1e-3 +#define BOARD_WIDTH_MINUS_WALL_EPS 9.999 + +#ifndef RANDOM_SEED +#define RANDOM_SEED 0 +#endif + +#ifndef BENCHMARK +#define BENCHMARK 0 +#endif + +using namespace std; + +alignas(64) vector pos_x, pos_y; +alignas(64) vector sb_sq; +alignas(64) vector sb_e_x, sb_e_y; +alignas(64) vector e_x, e_y; +alignas(64) vector curr_vel_x, curr_vel_y; +alignas(64) vector w_x, w_y; +alignas(64) vector desired_v0; +alignas(64) vector v_max; +alignas(64) vector destination_x, destination_y; +alignas(64) vector color; + +Pedestrian *init_pedestrians(int n_p) { + +#if RANDOM_SEED + random_device rd; + mt19937 gen(rd()); +#else + mt19937 gen(SEED); +#endif + + normal_distribution velocity_gen(v0_mean, v0_std_dev); + uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); + uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); + uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); + + const int n_p_pad = (n_p + 7) & ~7; + + pos_x.resize(n_p_pad); + pos_y.resize(n_p_pad); + sb_sq.resize(n_p_pad); + sb_e_x.resize(n_p_pad); + sb_e_y.resize(n_p_pad); + e_x.resize(n_p_pad); + e_y.resize(n_p_pad); + curr_vel_x.resize(n_p_pad); + curr_vel_y.resize(n_p_pad); + w_x.resize(n_p_pad); + w_y.resize(n_p_pad); + desired_v0.resize(n_p_pad); + v_max.resize(n_p_pad); + destination_x.resize(n_p_pad); + destination_y.resize(n_p_pad); + color.resize(n_p_pad); + + int n_p_half = n_p / 2; + + for (int i = 0; i < n_p_half; i++) { + double x = x_gen_left(gen); + double y = y_gen(gen); + + double dest_x = BOARD_LENGTH + 30.0; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 0; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int i = n_p_half; i < n_p; i++) { + double x = x_gen_right(gen); + double y = y_gen(gen); + + double dest_x = -30; + double dest_y = y; + + double init_e_x = dest_x - x; + double init_e_y = dest_y - y; + double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); + double init_e_unit_x = init_e_x / init_e_norm; + double init_e_unit_y = init_e_y / init_e_norm; + + double d_v0; + do { + d_v0 = velocity_gen(gen); + } while (d_v0 <= 0.1); + + double v_m = 1.3 * d_v0; + + color[i] = 1; + pos_x[i] = x; + pos_y[i] = y; + e_x[i] = init_e_unit_x; + e_y[i] = init_e_unit_y; + v_max[i] = v_m; + desired_v0[i] = d_v0; + destination_x[i] = dest_x; + destination_y[i] = dest_y; + + w_x[i] = 0.0; + w_y[i] = 0.0; + curr_vel_x[i] = 0.0; + curr_vel_y[i] = 0.0; + sb_sq[i] = 0.0; + sb_e_x[i] = 0.0; + sb_e_y[i] = 0.0; + } + + for (int j = n_p; j < n_p_pad; j++) { + pos_x[j] = 1e9; + pos_y[j] = 1e9; + sb_sq[j] = 0.0; + sb_e_x[j] = 0.0; + sb_e_y[j] = 0.0; + } + return nullptr; +} + +Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } + +static inline __m512d _avx512_exp_pd(__m512d x) { + + const __m512d log2e = _mm512_set1_pd(1.4426950408889634); + __m512d nf = _mm512_roundscale_pd(_mm512_mul_pd(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); + const __m512d ln2 = _mm512_set1_pd(0.6931471805599453); + __m512d r = _mm512_fnmadd_pd(nf, ln2, x); + + const __m512d c7 = _mm512_set1_pd(1.0 / 5040.0); + const __m512d c6 = _mm512_set1_pd(1.0 / 720.0); + const __m512d c5 = _mm512_set1_pd(1.0 / 120.0); + const __m512d c4 = _mm512_set1_pd(1.0 / 24.0); + const __m512d c3 = _mm512_set1_pd(1.0 / 6.0); + const __m512d c2 = _mm512_set1_pd(0.5); + const __m512d c1 = _mm512_set1_pd(1.0); + const __m512d c0 = _mm512_set1_pd(1.0); + + __m512d p = c7; + p = _mm512_fmadd_pd(p, r, c6); + p = _mm512_fmadd_pd(p, r, c5); + p = _mm512_fmadd_pd(p, r, c4); + p = _mm512_fmadd_pd(p, r, c3); + p = _mm512_fmadd_pd(p, r, c2); + p = _mm512_fmadd_pd(p, r, c1); + p = _mm512_fmadd_pd(p, r, c0); + + __m256d nf_lo = _mm512_castpd512_pd256(nf); + __m256d nf_hi = _mm512_extractf64x4_pd(nf, 1); + + __m128i ni32_lo = _mm256_cvtpd_epi32(nf_lo); + __m128i ni32_hi = _mm256_cvtpd_epi32(nf_hi); + + __m256i ni64_lo = _mm256_cvtepi32_epi64(ni32_lo); + __m256i ni64_hi = _mm256_cvtepi32_epi64(ni32_hi); + + __m512i ni64 = _mm512_inserti64x4(_mm512_castsi256_si512(ni64_lo), ni64_hi, 1); + + __m512i biased = _mm512_add_epi64(ni64, _mm512_set1_epi64(1023)); + + __m512i scaled = _mm512_slli_epi64(biased, 52); + + __m512d pow2n = _mm512_castsi512_pd(scaled); + + return _mm512_mul_pd(p, pow2n); +} + +static inline double fast_inv_sqrt(double number) { + double y = number; + + uint64_t i = bit_cast(y); + + i = 0x5fe6eb50c7b537a9ULL - (i >> 1); + + y = bit_cast(i); + + double x2 = number * 0.5; + y = y * (1.5 - (x2 * y * y)); + y = y * (1.5 - (x2 * y * y)); + y = y * (1.5 - (x2 * y * y)); + + return y; +} + +int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { + +#if BENCHMARK + (void)dump_path; +#endif + +#if !BENCHMARK + const int dump_interval = 1; + + vector> samples; + + if (dump_path) { + samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); + + for (int i = 0; i < n_p; i++) { + samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } + } +#endif + + const int n_p_pad = (n_p + 7) & ~7; + + alignas(64) constexpr double lane_offset_arr[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; + const __m512d lane_off = _mm512_load_pd(lane_offset_arr); + const __m512d v_neg_v0_div_2s = _mm512_set1_pd(NEG_V0_AB_div_TWO_SIGMA); + const __m512d v_neg_2s_inv = _mm512_set1_pd(NEG_TWO_SIGMA_INV); + const __m512d v_cos_phi_inv = _mm512_set1_pd(cos_phi_inv); + const __m512d v_dt_neg = _mm512_set1_pd(dt_neg); + const __m512d v_c_neg_dt = _mm512_set1_pd(c_neg_times_dt); + const __m512d v_zero = _mm512_setzero_pd(); + + const __m512d v_np = _mm512_set1_pd(n_p); + + for (int t = 0; t < n_iter; t++) { + + for (int i = 0; i < n_p; i++) { + double pos_x_i = pos_x[i]; + double pos_y_i = pos_y[i]; + double e_x_i = e_x[i]; + double e_y_i = e_y[i]; + double desired_v0_i = desired_v0[i]; + double curr_vel_x_i = curr_vel_x[i]; + double curr_vel_y_i = curr_vel_y[i]; + + double f_alpha_dt_x = (e_x_i * desired_v0_i - curr_vel_x_i) * dt_times_tau_alpha_inv; + double f_alpha_dt_y = (e_y_i * desired_v0_i - curr_vel_y_i) * dt_times_tau_alpha_inv; + + const __m512d pix = _mm512_set1_pd(pos_x_i); + const __m512d piy = _mm512_set1_pd(pos_y_i); + const __m512d eix = _mm512_set1_pd(e_x_i); + const __m512d eiy = _mm512_set1_pd(e_y_i); + const __m512d v_i = _mm512_set1_pd((double)i); + + __m512d acc_x = _mm512_setzero_pd(); + __m512d acc_y = _mm512_setzero_pd(); + + for (int j = 0; j < n_p_pad; j += 8) { + __m512d j_idx = _mm512_add_pd(_mm512_set1_pd((double)j), lane_off); + + __mmask8 m_neq_i = _mm512_cmp_pd_mask(j_idx, v_i, _CMP_NEQ_OQ); + __mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ); + __mmask8 active = m_neq_i & m_in_range; + + __m512d pjx = _mm512_load_pd(&pos_x[j]); + __m512d pjy = _mm512_load_pd(&pos_y[j]); + __m512d sbe_jx = _mm512_load_pd(&sb_e_x[j]); + __m512d sbe_jy = _mm512_load_pd(&sb_e_y[j]); + __m512d sb_jsq = _mm512_load_pd(&sb_sq[j]); + + __m512d r_ab_x = _mm512_sub_pd(pix, pjx); + __m512d r_ab_y = _mm512_sub_pd(piy, pjy); + __m512d r_ab_n2 = _mm512_fmadd_pd(r_ab_x, r_ab_x, _mm512_mul_pd(r_ab_y, r_ab_y)); + __m512d r_ab_n = _mm512_sqrt_pd(r_ab_n2); + + __m512d r_ab2_x = _mm512_sub_pd(r_ab_x, sbe_jx); + __m512d r_ab2_y = _mm512_sub_pd(r_ab_y, sbe_jy); + __m512d r_ab2_n2 = _mm512_fmadd_pd(r_ab2_x, r_ab2_x, _mm512_mul_pd(r_ab2_y, r_ab2_y)); + __m512d r_ab2_n = _mm512_sqrt_pd(r_ab2_n2); + + __m512d r_sum = _mm512_add_pd(r_ab_n, r_ab2_n); + __m512d two_b_sq = _mm512_fmsub_pd(r_sum, r_sum, sb_jsq); + __m512d two_b = _mm512_sqrt_pd(two_b_sq); + + __m512d e_arg = _mm512_mul_pd(two_b, v_neg_2s_inv); + __m512d e_val = _avx512_exp_pd(e_arg); + + __m512d numer = _mm512_mul_pd(_mm512_mul_pd(r_sum, v_neg_v0_div_2s), e_val); + __m512d denom = _mm512_mul_pd(_mm512_mul_pd(two_b, r_ab_n), r_ab2_n); + __m512d denom_safe = _mm512_mask_blend_pd(active, _mm512_set1_pd(1.0), denom); + __m512d smc = _mm512_div_pd(numer, denom_safe); + + __m512d sm_r_ab = _mm512_mul_pd(smc, r_ab2_n); + __m512d sm_r_ab2 = _mm512_mul_pd(smc, r_ab_n); + + __m512d f_x = _mm512_fmadd_pd(r_ab2_x, sm_r_ab2, _mm512_mul_pd(r_ab_x, sm_r_ab)); + __m512d f_y = _mm512_fmadd_pd(r_ab2_y, sm_r_ab2, _mm512_mul_pd(r_ab_y, sm_r_ab)); + + __m512d a = _mm512_mul_pd(_mm512_fmadd_pd(eix, f_x, _mm512_mul_pd(eiy, f_y)), v_cos_phi_inv); + __m512d a_sq = _mm512_mul_pd(a, a); + __m512d b_sq = _mm512_fmadd_pd(f_x, f_x, _mm512_mul_pd(f_y, f_y)); + + __mmask8 k_ge = _mm512_cmp_pd_mask(a, v_zero, _CMP_GE_OQ); + __mmask8 k_gt = _mm512_cmp_pd_mask(a_sq, b_sq, _CMP_GT_OQ); + __mmask8 k_cond = _mm512_kand(k_ge, k_gt); + __m512d w_val = _mm512_mask_blend_pd(k_cond, v_dt_neg, v_c_neg_dt); + + __m512d f_x_safe = _mm512_maskz_mov_pd(active, f_x); + __m512d f_y_safe = _mm512_maskz_mov_pd(active, f_y); + acc_x = _mm512_fmadd_pd(f_x_safe, w_val, acc_x); + acc_y = _mm512_fmadd_pd(f_y_safe, w_val, acc_y); + } + + f_alpha_dt_x += _mm512_reduce_add_pd(acc_x); + f_alpha_dt_y += _mm512_reduce_add_pd(acc_y); + + for (int k = 0; k < board.n_obstacles; k++) { + + double obs_start_x = board.obstacles[k].start_x; + double obs_start_y = board.obstacles[k].start_y; + double obs_v_x = board.obstacles[k].obstacle_v_x; + double obs_v_y = board.obstacles[k].obstacle_v_y; + double obs_length_sq_inv = board.obstacles[k].length_sq_inv; + + double v1_x = pos_x_i - obs_start_x; + double v1_y = pos_y_i - obs_start_y; + + double projection = (v1_x * obs_v_x + v1_y * obs_v_y) * obs_length_sq_inv; + + if (projection < 0.0) + projection = 0.0; + else if (projection > 1.0) + projection = 1.0; + + double r_aB_x = pos_x_i - (obs_start_x + obs_v_x * projection); + double r_aB_y = pos_y_i - (obs_start_y + obs_v_y * projection); + + double r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); + double scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; + double f_alpha_B_neg_x = r_aB_x * scalar_mult; + double f_alpha_B_neg_y = r_aB_y * scalar_mult; + + double w_val = dt_neg; + double e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y_i : e_y_i); + if (e_dot_f < cos_phi) + w_val = c_neg_times_dt; + + f_alpha_dt_x += f_alpha_B_neg_x * w_val; + f_alpha_dt_y += f_alpha_B_neg_y * w_val; + } + + w_x[i] += f_alpha_dt_x; + w_y[i] += f_alpha_dt_y; + } + + for (int i = 0; i < n_p; i++) { + double color_i = color[i]; + double curr_vel_x_i = curr_vel_x[i]; + double curr_vel_y_i = curr_vel_y[i]; + double w_x_i = w_x[i]; + double w_y_i = w_y[i]; + double pos_x_i = pos_x[i]; + double pos_y_i = pos_y[i]; + double e_x_i = e_x[i]; + double e_y_i = e_y[i]; + double v_max_i = v_max[i]; + double desired_v0_i = desired_v0[i]; + double destination_x_i = destination_x[i]; + double destination_y_i = destination_y[i]; + + double w_norm_inv = fast_inv_sqrt(w_x_i * w_x_i + w_y_i * w_y_i); + + double correction_factor = min(1.0, v_max_i * w_norm_inv); + + curr_vel_x_i = w_x_i * correction_factor; + curr_vel_y_i = w_y_i * correction_factor; + + pos_x_i = pos_x_i + curr_vel_x_i * dt; + pos_y_i = pos_y_i + curr_vel_y_i * dt; + + if (!color_i && pos_x_i > (BOARD_LENGTH - 1.5)) { + pos_x_i = 1.0; + destination_y_i = pos_y_i; + w_x_i = desired_v0_i; + w_y_i = 0.0; + curr_vel_x_i = w_x_i; + curr_vel_y_i = w_y_i; + } + if (color_i && pos_x_i < 1.5) { + pos_x_i = BOARD_LENGTH - 1.0; + destination_y_i = pos_y_i; + w_x_i = -desired_v0_i; + w_y_i = 0.0; + curr_vel_x_i = w_x_i; + curr_vel_y_i = w_y_i; + } + + pos_y_i = clamp(pos_y_i, WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); + + double ex = destination_x_i - pos_x_i; + double ey = destination_y_i - pos_y_i; + + double e_norm_inv = fast_inv_sqrt(ex * ex + ey * ey); + + e_x_i = ex * e_norm_inv; + e_y_i = ey * e_norm_inv; + + double velocity_norm_sq = curr_vel_x_i * curr_vel_x_i + curr_vel_y_i * curr_vel_y_i; + double sb_sq_i = velocity_norm_sq * delta_t_sq; + double velocity_norm_mul_delta_t = sqrt(sb_sq_i); + + curr_vel_x[i] = curr_vel_x_i; + curr_vel_y[i] = curr_vel_y_i; + + w_x[i] = w_x_i; + w_y[i] = w_y_i; + + pos_x[i] = pos_x_i; + pos_y[i] = pos_y_i; + + e_x[i] = e_x_i; + e_y[i] = e_y_i; + + destination_y[i] = destination_y_i; + + sb_sq[i] = sb_sq_i; + sb_e_x[i] = e_x_i * velocity_norm_mul_delta_t; + sb_e_y[i] = e_y_i * velocity_norm_mul_delta_t; + +#if !BENCHMARK + if (dump_path && (t % dump_interval == 0)) { + samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); + } +#endif + } + } + +#if !BENCHMARK + if (dump_path) { + ofstream out(dump_path); + out << "time,id,x,y,p.e.x,p.e.y,color\n"; + + for (auto &[t, id, x, y, e_x, e_y, c] : samples) { + out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; + } + } +#endif + + return 0; +}