// # Changes: Vectorised second pass with AVX-512 // # Predecessor: variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp #include "../headers/board_reduced.hpp" #include "../headers/constants.hpp" #include "../headers/pedestrian_reduced.hpp" #include #include #include #define SEED 42 #define BOARD_WIDTH 10.0 #define BOARD_LENGTH 50.0 #define WALL_EPS 1e-3 #define BOARD_WIDTH_MINUS_WALL_EPS 9.999 #ifndef RANDOM_SEED #define RANDOM_SEED 0 #endif #ifndef BENCHMARK #define BENCHMARK 0 #endif using namespace std; alignas(64) vector pos_x, pos_y; alignas(64) vector sb_sq; alignas(64) vector sb_e_x, sb_e_y; alignas(64) vector e_x, e_y; alignas(64) vector curr_vel_x, curr_vel_y; alignas(64) vector w_x, w_y; alignas(64) vector desired_v0; alignas(64) vector v_max; alignas(64) vector destination_x, destination_y; alignas(64) vector color; Pedestrian *init_pedestrians(int n_p) { #if RANDOM_SEED random_device rd; mt19937 gen(rd()); #else mt19937 gen(SEED); #endif normal_distribution velocity_gen(v0_mean, v0_std_dev); uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); const int n_p_pad = (n_p + 7) & ~7; pos_x.resize(n_p_pad); pos_y.resize(n_p_pad); sb_sq.resize(n_p_pad); sb_e_x.resize(n_p_pad); sb_e_y.resize(n_p_pad); e_x.resize(n_p_pad); e_y.resize(n_p_pad); curr_vel_x.resize(n_p_pad); curr_vel_y.resize(n_p_pad); w_x.resize(n_p_pad); w_y.resize(n_p_pad); desired_v0.resize(n_p_pad); v_max.resize(n_p_pad); destination_x.resize(n_p_pad); destination_y.resize(n_p_pad); color.resize(n_p_pad); int n_p_half = n_p / 2; for (int i = 0; i < n_p_half; i++) { double x = x_gen_left(gen); double y = y_gen(gen); double dest_x = BOARD_LENGTH + 30.0; double dest_y = y; double init_e_x = dest_x - x; double init_e_y = dest_y - y; double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); double init_e_unit_x = init_e_x / init_e_norm; double init_e_unit_y = init_e_y / init_e_norm; double d_v0; do { d_v0 = velocity_gen(gen); } while (d_v0 <= 0.1); double v_m = 1.3 * d_v0; color[i] = 0; pos_x[i] = x; pos_y[i] = y; e_x[i] = init_e_unit_x; e_y[i] = init_e_unit_y; v_max[i] = v_m; desired_v0[i] = d_v0; destination_x[i] = dest_x; destination_y[i] = dest_y; w_x[i] = 0.0; w_y[i] = 0.0; curr_vel_x[i] = 0.0; curr_vel_y[i] = 0.0; sb_sq[i] = 0.0; sb_e_x[i] = 0.0; sb_e_y[i] = 0.0; } for (int i = n_p_half; i < n_p; i++) { double x = x_gen_right(gen); double y = y_gen(gen); double dest_x = -30; double dest_y = y; double init_e_x = dest_x - x; double init_e_y = dest_y - y; double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); double init_e_unit_x = init_e_x / init_e_norm; double init_e_unit_y = init_e_y / init_e_norm; double d_v0; do { d_v0 = velocity_gen(gen); } while (d_v0 <= 0.1); double v_m = 1.3 * d_v0; color[i] = 1; pos_x[i] = x; pos_y[i] = y; e_x[i] = init_e_unit_x; e_y[i] = init_e_unit_y; v_max[i] = v_m; desired_v0[i] = d_v0; destination_x[i] = dest_x; destination_y[i] = dest_y; w_x[i] = 0.0; w_y[i] = 0.0; curr_vel_x[i] = 0.0; curr_vel_y[i] = 0.0; sb_sq[i] = 0.0; sb_e_x[i] = 0.0; sb_e_y[i] = 0.0; } for (int j = n_p; j < n_p_pad; j++) { pos_x[j] = 1e9; pos_y[j] = 1e9; sb_sq[j] = 0.0; sb_e_x[j] = 0.0; sb_e_y[j] = 0.0; } return nullptr; } Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } static inline __m512d _avx512_exp_pd(__m512d x) { const __m512d log2e = _mm512_set1_pd(1.4426950408889634); __m512d nf = _mm512_roundscale_pd(_mm512_mul_pd(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); const __m512d ln2 = _mm512_set1_pd(0.6931471805599453); __m512d r = _mm512_fnmadd_pd(nf, ln2, x); const __m512d c7 = _mm512_set1_pd(1.0 / 5040.0); const __m512d c6 = _mm512_set1_pd(1.0 / 720.0); const __m512d c5 = _mm512_set1_pd(1.0 / 120.0); const __m512d c4 = _mm512_set1_pd(1.0 / 24.0); const __m512d c3 = _mm512_set1_pd(1.0 / 6.0); const __m512d c2 = _mm512_set1_pd(0.5); const __m512d c1 = _mm512_set1_pd(1.0); const __m512d c0 = _mm512_set1_pd(1.0); __m512d p = c7; p = _mm512_fmadd_pd(p, r, c6); p = _mm512_fmadd_pd(p, r, c5); p = _mm512_fmadd_pd(p, r, c4); p = _mm512_fmadd_pd(p, r, c3); p = _mm512_fmadd_pd(p, r, c2); p = _mm512_fmadd_pd(p, r, c1); p = _mm512_fmadd_pd(p, r, c0); __m256d nf_lo = _mm512_castpd512_pd256(nf); __m256d nf_hi = _mm512_extractf64x4_pd(nf, 1); __m128i ni32_lo = _mm256_cvtpd_epi32(nf_lo); __m128i ni32_hi = _mm256_cvtpd_epi32(nf_hi); __m256i ni64_lo = _mm256_cvtepi32_epi64(ni32_lo); __m256i ni64_hi = _mm256_cvtepi32_epi64(ni32_hi); __m512i ni64 = _mm512_inserti64x4(_mm512_castsi256_si512(ni64_lo), ni64_hi, 1); __m512i biased = _mm512_add_epi64(ni64, _mm512_set1_epi64(1023)); __m512i scaled = _mm512_slli_epi64(biased, 52); __m512d pow2n = _mm512_castsi512_pd(scaled); return _mm512_mul_pd(p, pow2n); } int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { #if BENCHMARK (void)dump_path; #endif #if !BENCHMARK const int dump_interval = 1; vector> samples; if (dump_path) { samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); for (int i = 0; i < n_p; i++) { samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); } } #endif const int n_p_pad = (n_p + 7) & ~7; alignas(64) constexpr double lane_offset_arr[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; const __m512d lane_off = _mm512_load_pd(lane_offset_arr); const __m512d v_neg_v0_div_2s = _mm512_set1_pd(NEG_V0_AB_div_TWO_SIGMA); const __m512d v_neg_2s_inv = _mm512_set1_pd(NEG_TWO_SIGMA_INV); const __m512d v_cos_phi_inv = _mm512_set1_pd(cos_phi_inv); const __m512d v_dt_neg = _mm512_set1_pd(dt_neg); const __m512d v_c_neg_dt = _mm512_set1_pd(c_neg_times_dt); const __m512d v_zero = _mm512_setzero_pd(); const __m512d v_np = _mm512_set1_pd(n_p); for (int t = 0; t < n_iter; t++) { for (int i = 0; i < n_p; i++) { double pos_x_i = pos_x[i]; double pos_y_i = pos_y[i]; double e_x_i = e_x[i]; double e_y_i = e_y[i]; double desired_v0_i = desired_v0[i]; double curr_vel_x_i = curr_vel_x[i]; double curr_vel_y_i = curr_vel_y[i]; double f_alpha_dt_x = (e_x_i * desired_v0_i - curr_vel_x_i) * dt_times_tau_alpha_inv; double f_alpha_dt_y = (e_y_i * desired_v0_i - curr_vel_y_i) * dt_times_tau_alpha_inv; const __m512d pix = _mm512_set1_pd(pos_x_i); const __m512d piy = _mm512_set1_pd(pos_y_i); const __m512d eix = _mm512_set1_pd(e_x_i); const __m512d eiy = _mm512_set1_pd(e_y_i); const __m512d v_i = _mm512_set1_pd((double)i); __m512d acc_x = _mm512_setzero_pd(); __m512d acc_y = _mm512_setzero_pd(); for (int j = 0; j < n_p_pad; j += 8) { __m512d j_idx = _mm512_add_pd(_mm512_set1_pd((double)j), lane_off); __mmask8 m_neq_i = _mm512_cmp_pd_mask(j_idx, v_i, _CMP_NEQ_OQ); __mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ); __mmask8 active = m_neq_i & m_in_range; __m512d pjx = _mm512_loadu_pd(&pos_x[j]); __m512d pjy = _mm512_loadu_pd(&pos_y[j]); __m512d sbe_jx = _mm512_loadu_pd(&sb_e_x[j]); __m512d sbe_jy = _mm512_loadu_pd(&sb_e_y[j]); __m512d sb_jsq = _mm512_loadu_pd(&sb_sq[j]); __m512d r_ab_x = _mm512_sub_pd(pix, pjx); __m512d r_ab_y = _mm512_sub_pd(piy, pjy); __m512d r_ab_n2 = _mm512_fmadd_pd(r_ab_x, r_ab_x, _mm512_mul_pd(r_ab_y, r_ab_y)); __m512d r_ab_n = _mm512_sqrt_pd(r_ab_n2); __m512d r_ab2_x = _mm512_sub_pd(r_ab_x, sbe_jx); __m512d r_ab2_y = _mm512_sub_pd(r_ab_y, sbe_jy); __m512d r_ab2_n2 = _mm512_fmadd_pd(r_ab2_x, r_ab2_x, _mm512_mul_pd(r_ab2_y, r_ab2_y)); __m512d r_ab2_n = _mm512_sqrt_pd(r_ab2_n2); __m512d r_sum = _mm512_add_pd(r_ab_n, r_ab2_n); __m512d two_b_sq = _mm512_fmsub_pd(r_sum, r_sum, sb_jsq); __m512d two_b = _mm512_sqrt_pd(two_b_sq); __m512d e_arg = _mm512_mul_pd(two_b, v_neg_2s_inv); __m512d e_val = _avx512_exp_pd(e_arg); __m512d numer = _mm512_mul_pd(_mm512_mul_pd(r_sum, v_neg_v0_div_2s), e_val); __m512d denom = _mm512_mul_pd(_mm512_mul_pd(two_b, r_ab_n), r_ab2_n); __m512d smc = _mm512_div_pd(numer, denom); __m512d sm_r_ab = _mm512_mul_pd(smc, r_ab2_n); __m512d sm_r_ab2 = _mm512_mul_pd(smc, r_ab_n); __m512d f_x = _mm512_fmadd_pd(r_ab2_x, sm_r_ab2, _mm512_mul_pd(r_ab_x, sm_r_ab)); __m512d f_y = _mm512_fmadd_pd(r_ab2_y, sm_r_ab2, _mm512_mul_pd(r_ab_y, sm_r_ab)); __m512d a = _mm512_mul_pd(_mm512_fmadd_pd(eix, f_x, _mm512_mul_pd(eiy, f_y)), v_cos_phi_inv); __m512d a_sq = _mm512_mul_pd(a, a); __m512d b_sq = _mm512_fmadd_pd(f_x, f_x, _mm512_mul_pd(f_y, f_y)); __mmask8 k_ge = _mm512_cmp_pd_mask(a, v_zero, _CMP_GE_OQ); __mmask8 k_gt = _mm512_cmp_pd_mask(a_sq, b_sq, _CMP_GT_OQ); __mmask8 k_cond = _mm512_kand(k_ge, k_gt); __m512d w_val = _mm512_mask_blend_pd(k_cond, v_dt_neg, v_c_neg_dt); __m512d f_x_safe = _mm512_maskz_mov_pd(active, f_x); __m512d f_y_safe = _mm512_maskz_mov_pd(active, f_y); acc_x = _mm512_fmadd_pd(f_x_safe, w_val, acc_x); acc_y = _mm512_fmadd_pd(f_y_safe, w_val, acc_y); } f_alpha_dt_x += _mm512_reduce_add_pd(acc_x); f_alpha_dt_y += _mm512_reduce_add_pd(acc_y); for (int k = 0; k < board.n_obstacles; k++) { double v1_x = pos_x_i - board.obstacles[k].start_x; double v1_y = pos_y_i - board.obstacles[k].start_y; double projection = (v1_x * board.obstacles[k].obstacle_v_x + v1_y * board.obstacles[k].obstacle_v_y) * board.obstacles[k].length_sq_inv; if (projection < 0.0) projection = 0.0; else if (projection > 1.0) projection = 1.0; double r_aB_x = pos_x_i - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); double r_aB_y = pos_y_i - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); double r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); double scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; double f_alpha_B_neg_x = r_aB_x * scalar_mult; double f_alpha_B_neg_y = r_aB_y * scalar_mult; double w_val = dt_neg; double e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y_i : e_y_i); if (e_dot_f < cos_phi) w_val = c_neg_times_dt; f_alpha_dt_x += f_alpha_B_neg_x * w_val; f_alpha_dt_y += f_alpha_B_neg_y * w_val; } w_x[i] += f_alpha_dt_x; w_y[i] += f_alpha_dt_y; } for (int i = 0; i < n_p_pad; i += 8) { __m512d w_x_i_vec = _mm512_loadu_pd(&w_x[i]); __m512d w_y_i_vec = _mm512_loadu_pd(&w_y[i]); __m512d v_max_i_vec = _mm512_loadu_pd(&v_max[i]); __m512d pos_x_i_vec = _mm512_loadu_pd(&pos_x[i]); __m512d pos_y_i_vec = _mm512_loadu_pd(&pos_y[i]); __m512d destination_y_i_vec = _mm512_loadu_pd(&destination_y[i]); __m512d destination_x_i_vec = _mm512_loadu_pd(&destination_x[i]); __m512d desired_v0_i_vec = _mm512_loadu_pd(&desired_v0[i]); __m256i color_i_vec = _mm256_loadu_si256((__m256i *)&color[i]); __m512d w_norm_sq = _mm512_fmadd_pd(w_x_i_vec, w_x_i_vec, _mm512_mul_pd(w_y_i_vec, w_y_i_vec)); __m512d w_norm = _mm512_sqrt_pd(w_norm_sq); __m512d correction_factor = _mm512_min_pd(_mm512_set1_pd(1.0), _mm512_div_pd(v_max_i_vec, w_norm)); __m512d curr_vel_x_i_vec = _mm512_mul_pd(w_x_i_vec, correction_factor); __m512d curr_vel_y_i_vec = _mm512_mul_pd(w_y_i_vec, correction_factor); __m512d dt_vec = _mm512_set1_pd(dt); pos_x_i_vec = _mm512_fmadd_pd(curr_vel_x_i_vec, dt_vec, pos_x_i_vec); pos_y_i_vec = _mm512_fmadd_pd(curr_vel_y_i_vec, dt_vec, pos_y_i_vec); __mmask8 cond_1_left = _mm256_cmpeq_epi32_mask(color_i_vec, _mm256_setzero_si256()); __mmask8 cond_1_right = _mm512_cmp_pd_mask(pos_x_i_vec, _mm512_set1_pd(BOARD_LENGTH - 1.5), _CMP_GT_OQ); __mmask8 cond_1 = cond_1_left & cond_1_right; pos_x_i_vec = _mm512_mask_blend_pd(cond_1, pos_x_i_vec, _mm512_set1_pd(1.0)); destination_y_i_vec = _mm512_mask_blend_pd(cond_1, destination_y_i_vec, pos_y_i_vec); w_x_i_vec = _mm512_mask_blend_pd(cond_1, w_x_i_vec, desired_v0_i_vec); w_y_i_vec = _mm512_mask_blend_pd(cond_1, w_y_i_vec, _mm512_setzero_pd()); curr_vel_x_i_vec = _mm512_mask_blend_pd(cond_1, curr_vel_x_i_vec, w_x_i_vec); curr_vel_y_i_vec = _mm512_mask_blend_pd(cond_1, curr_vel_y_i_vec, w_y_i_vec); __mmask8 cond_2_left = _mm256_cmpeq_epi32_mask(color_i_vec, _mm256_set1_epi32(1)); __mmask8 cond_2_right = _mm512_cmp_pd_mask(pos_x_i_vec, _mm512_set1_pd(1.5), _CMP_LT_OQ); __mmask8 cond_2 = cond_2_left & cond_2_right; pos_x_i_vec = _mm512_mask_blend_pd(cond_2, pos_x_i_vec, _mm512_set1_pd(BOARD_LENGTH - 1.0)); destination_y_i_vec = _mm512_mask_blend_pd(cond_2, destination_y_i_vec, pos_y_i_vec); __m512d neg_desired_v0 = _mm512_xor_pd(desired_v0_i_vec, _mm512_set1_pd(-0.0)); w_x_i_vec = _mm512_mask_blend_pd(cond_2, w_x_i_vec, neg_desired_v0); w_y_i_vec = _mm512_mask_blend_pd(cond_2, w_y_i_vec, _mm512_setzero_pd()); curr_vel_x_i_vec = _mm512_mask_blend_pd(cond_2, curr_vel_x_i_vec, w_x_i_vec); curr_vel_y_i_vec = _mm512_mask_blend_pd(cond_2, curr_vel_y_i_vec, w_y_i_vec); pos_y_i_vec = _mm512_max_pd(pos_y_i_vec, _mm512_set1_pd(WALL_EPS)); pos_y_i_vec = _mm512_min_pd(pos_y_i_vec, _mm512_set1_pd(BOARD_WIDTH_MINUS_WALL_EPS)); __m512d ex_vec = _mm512_sub_pd(destination_x_i_vec, pos_x_i_vec); __m512d ey_vec = _mm512_sub_pd(destination_y_i_vec, pos_y_i_vec); __m512d e_norm_sq = _mm512_fmadd_pd(ex_vec, ex_vec, _mm512_mul_pd(ey_vec, ey_vec)); __m512d e_norm = _mm512_sqrt_pd(e_norm_sq); __m512d e_norm_inv = _mm512_div_pd(_mm512_set1_pd(1.0), e_norm); __m512d e_x_i_vec = _mm512_mul_pd(ex_vec, e_norm_inv); __m512d e_y_i_vec = _mm512_mul_pd(ey_vec, e_norm_inv); __m512d vel_norm_sq = _mm512_fmadd_pd(curr_vel_x_i_vec, curr_vel_x_i_vec, _mm512_mul_pd(curr_vel_y_i_vec, curr_vel_y_i_vec)); __m512d sb_sq_i_vec = _mm512_mul_pd(vel_norm_sq, _mm512_set1_pd(delta_t_sq)); __m512d vel_norm_dt = _mm512_sqrt_pd(sb_sq_i_vec); _mm512_storeu_pd(&curr_vel_x[i], curr_vel_x_i_vec); _mm512_storeu_pd(&curr_vel_y[i], curr_vel_y_i_vec); _mm512_storeu_pd(&w_x[i], w_x_i_vec); _mm512_storeu_pd(&w_y[i], w_y_i_vec); _mm512_storeu_pd(&pos_x[i], pos_x_i_vec); _mm512_storeu_pd(&pos_y[i], pos_y_i_vec); _mm512_storeu_pd(&e_x[i], e_x_i_vec); _mm512_storeu_pd(&e_y[i], e_y_i_vec); _mm512_storeu_pd(&destination_y[i], destination_y_i_vec); _mm512_storeu_pd(&sb_sq[i], sb_sq_i_vec); _mm512_storeu_pd(&sb_e_x[i], _mm512_mul_pd(e_x_i_vec, vel_norm_dt)); _mm512_storeu_pd(&sb_e_y[i], _mm512_mul_pd(e_y_i_vec, vel_norm_dt)); } #if !BENCHMARK for (int i = 0; i < n_p; i++) { if (dump_path && (t % dump_interval == 0)) { samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); } } #endif } #if !BENCHMARK if (dump_path) { ofstream out(dump_path); out << "time,id,x,y,p.e.x,p.e.y,color\n"; for (auto &[t, id, x, y, e_x, e_y, c] : samples) { out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; } } #endif return 0; }