// # Changes: pure vrsqrtps // # Predecessor: variant_float_inlining_SoA_AVX2_polyexp.cpp // // We first tried rsqrt + 1 Newton step to keep full sqrt precision, but it // turned out slower than just using vsqrtps. The Newton step needs 2 muls // and an fnmadd on top of rsqrt, so the whole recovered sqrt ends up at // ~20 cycles of latency, while vsqrtps is only 12. Since the OoO core was // already running multiple sqrts in parallel, sqrt throughput wasn't really // the bottleneck. All the Newton step did was make the dependency chain // longer, which hurt more than the faster rsqrt helped. // // So we drop the Newton step. rsqrt + 1 mul gives us a sqrt in 8 cycles // with about 12-bit precision, which is well within the 1cm verify tolerance. // The main structural win is killing the vdivps. rsqrt produces 1/sqrt(x) // directly, so we keep the inverses around (inv_r_ab_n, inv_r_ab2_n, // inv_two_b) and rewrite the force scaling. The original had // "numer / (two_b * r_ab_n * r_ab2_n)" which needs a vdivps; now it's // "numer * inv_two_b * inv_r_ab_n * inv_r_ab2_n", trading one divide for // two muls. Divide was the long-pole op on port 0, so this is the bigger // of the two changes. #include "../headers/board_reduced_float.hpp" #include "../headers/constants_float.hpp" #include "../headers/pedestrian_reduced_float.hpp" #include // avx2 #include #include #include #include #include #include #include #include #define SEED 42 #define BOARD_WIDTH 10.0f #define BOARD_LENGTH 50.0f #define WALL_EPS 1e-3f #define BOARD_WIDTH_MINUS_WALL_EPS 9.999f #ifndef RANDOM_SEED #define RANDOM_SEED 0 #endif #ifndef BENCHMARK #define BENCHMARK 0 #endif using namespace std; vector pos_x, pos_y; vector sb_sq; vector sb_e_x, sb_e_y; vector e_x, e_y; vector curr_vel_x, curr_vel_y; vector w_x, w_y; vector desired_v0; vector v_max; vector destination_x, destination_y; vector color; Pedestrian *init_pedestrians(int n_p) { #if RANDOM_SEED random_device rd; mt19937 gen(rd()); #else mt19937 gen(SEED); #endif normal_distribution velocity_gen(v0_mean, v0_std_dev); uniform_real_distribution y_gen(0.3, BOARD_WIDTH - 0.3); uniform_real_distribution x_gen_left(2.0, BOARD_LENGTH / 2.0); uniform_real_distribution x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0); const int n_p_pad = (n_p + 7) & ~7; pos_x.resize(n_p_pad); pos_y.resize(n_p_pad); sb_sq.resize(n_p_pad); sb_e_x.resize(n_p_pad); sb_e_y.resize(n_p_pad); e_x.resize(n_p_pad); e_y.resize(n_p_pad); curr_vel_x.resize(n_p_pad); curr_vel_y.resize(n_p_pad); w_x.resize(n_p_pad); w_y.resize(n_p_pad); desired_v0.resize(n_p_pad); v_max.resize(n_p_pad); destination_x.resize(n_p_pad); destination_y.resize(n_p_pad); color.resize(n_p_pad); int n_p_half = n_p / 2; for (int i = 0; i < n_p_half; i++) { float x = x_gen_left(gen); float y = y_gen(gen); float dest_x = BOARD_LENGTH + 30.0; float dest_y = y; float init_e_x = dest_x - x; float init_e_y = dest_y - y; float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); float init_e_unit_x = init_e_x / init_e_norm; float init_e_unit_y = init_e_y / init_e_norm; float d_v0; do { d_v0 = velocity_gen(gen); } while (d_v0 <= 0.1); float v_m = 1.3 * d_v0; color[i] = 0; pos_x[i] = x; pos_y[i] = y; e_x[i] = init_e_unit_x; e_y[i] = init_e_unit_y; v_max[i] = v_m; desired_v0[i] = d_v0; destination_x[i] = dest_x; destination_y[i] = dest_y; w_x[i] = 0.0f; w_y[i] = 0.0f; curr_vel_x[i] = 0.0f; curr_vel_y[i] = 0.0f; sb_sq[i] = 0.0f; sb_e_x[i] = 0.0f; sb_e_y[i] = 0.0f; } for (int i = n_p_half; i < n_p; i++) { float x = x_gen_right(gen); float y = y_gen(gen); float dest_x = -30; float dest_y = y; float init_e_x = dest_x - x; float init_e_y = dest_y - y; float init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y); float init_e_unit_x = init_e_x / init_e_norm; float init_e_unit_y = init_e_y / init_e_norm; float d_v0; do { d_v0 = velocity_gen(gen); } while (d_v0 <= 0.1); float v_m = 1.3 * d_v0; color[i] = 1; pos_x[i] = x; pos_y[i] = y; e_x[i] = init_e_unit_x; e_y[i] = init_e_unit_y; v_max[i] = v_m; desired_v0[i] = d_v0; destination_x[i] = dest_x; destination_y[i] = dest_y; w_x[i] = 0.0f; w_y[i] = 0.0f; curr_vel_x[i] = 0.0f; curr_vel_y[i] = 0.0f; sb_sq[i] = 0.0f; sb_e_x[i] = 0.0f; sb_e_y[i] = 0.0f; } for (int j = n_p; j < n_p_pad; j++) { pos_x[j] = 1e9f; pos_y[j] = 1e9f; sb_sq[j] = 0.0f; sb_e_x[j] = 0.0f; sb_e_y[j] = 0.0f; } return nullptr; } Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); } // simd expf, same as the predecessor. degree-5 taylor on the [-ln2/2, ln2/2] reduced range, // schraudolph 2^n bit trick on the exponent slot. static inline __m256 exp256_ps(__m256 x) { const __m256 log2e = _mm256_set1_ps(1.4426950408889634f); __m256 nf = _mm256_round_ps(_mm256_mul_ps(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); const __m256 ln2 = _mm256_set1_ps(0.6931471805599453f); __m256 r = _mm256_fnmadd_ps(nf, ln2, x); const __m256 c5 = _mm256_set1_ps(1.0f / 120.0f); const __m256 c4 = _mm256_set1_ps(1.0f / 24.0f); const __m256 c3 = _mm256_set1_ps(1.0f / 6.0f); const __m256 c2 = _mm256_set1_ps(0.5f); const __m256 c1 = _mm256_set1_ps(1.0f); const __m256 c0 = _mm256_set1_ps(1.0f); __m256 p = c5; p = _mm256_fmadd_ps(p, r, c4); p = _mm256_fmadd_ps(p, r, c3); p = _mm256_fmadd_ps(p, r, c2); p = _mm256_fmadd_ps(p, r, c1); p = _mm256_fmadd_ps(p, r, c0); __m256i ni = _mm256_cvtps_epi32(nf); __m256i biased = _mm256_add_epi32(ni, _mm256_set1_epi32(127)); __m256i scaled = _mm256_slli_epi32(biased, 23); __m256 pow2n = _mm256_castsi256_ps(scaled); return _mm256_mul_ps(p, pow2n); } int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) { (void)pedestrians; #if BENCHMARK (void)dump_path; #endif #if !BENCHMARK const int dump_interval = 1; vector> samples; if (dump_path) { samples.reserve(static_cast(n_iter / dump_interval + 1) * n_p); for (int i = 0; i < n_p; i++) { samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); } } #endif const int n_p_pad = (n_p + 7) & ~7; alignas(32) const float lane_offset_arr[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}; const __m256 lane_off = _mm256_load_ps(lane_offset_arr); const __m256 v_neg_v0_div_2s = _mm256_set1_ps(NEG_V0_AB_div_TWO_SIGMA); const __m256 v_neg_2s_inv = _mm256_set1_ps(NEG_TWO_SIGMA_INV); const __m256 v_cos_phi_sq = _mm256_set1_ps(cos_phi_sq); const __m256 v_dt_neg = _mm256_set1_ps(dt_neg); const __m256 v_c_neg_dt = _mm256_set1_ps(c_neg_times_dt); const __m256 v_zero = _mm256_setzero_ps(); const __m256 v_eps_sq = _mm256_set1_ps(EPS_SQ); const __m256 v_np = _mm256_set1_ps((float)n_p); const __m256 v_exp_min = _mm256_set1_ps(-87.0f); for (int t = 0; t < n_iter; t++) { for (int i = 0; i < n_p; i++) { float desired_velocity_x = e_x[i] * desired_v0[i]; float desired_velocity_y = e_y[i] * desired_v0[i]; float f_alpha_dt_x = (desired_velocity_x - curr_vel_x[i]) * dt_times_tau_alpha_inv; float f_alpha_dt_y = (desired_velocity_y - curr_vel_y[i]) * dt_times_tau_alpha_inv; const __m256 pix = _mm256_set1_ps(pos_x[i]); const __m256 piy = _mm256_set1_ps(pos_y[i]); const __m256 eix = _mm256_set1_ps(e_x[i]); const __m256 eiy = _mm256_set1_ps(e_y[i]); const __m256 v_i = _mm256_set1_ps((float)i); __m256 acc_x = _mm256_setzero_ps(); __m256 acc_y = _mm256_setzero_ps(); for (int j = 0; j < n_p_pad; j += 8) { __m256 j_idx = _mm256_add_ps(_mm256_set1_ps((float)j), lane_off); __m256 m_neq_i = _mm256_cmp_ps(j_idx, v_i, _CMP_NEQ_OQ); __m256 m_in_range = _mm256_cmp_ps(j_idx, v_np, _CMP_LT_OQ); __m256 active = _mm256_and_ps(m_neq_i, m_in_range); __m256 pjx = _mm256_loadu_ps(&pos_x[j]); __m256 pjy = _mm256_loadu_ps(&pos_y[j]); __m256 sbe_jx = _mm256_loadu_ps(&sb_e_x[j]); __m256 sbe_jy = _mm256_loadu_ps(&sb_e_y[j]); __m256 sb_jsq = _mm256_loadu_ps(&sb_sq[j]); __m256 r_ab_x = _mm256_sub_ps(pix, pjx); __m256 r_ab_y = _mm256_sub_ps(piy, pjy); __m256 r_ab_n2 = _mm256_fmadd_ps(r_ab_x, r_ab_x, _mm256_mul_ps(r_ab_y, r_ab_y)); // clamp before rsqrt: rsqrt(0) = inf, then x*inf = 0*inf = NaN downstream. EPS_SQ // means the inv_sqrt tops out at 1/EPS and r_ab_n = EPS_SQ/EPS = EPS, matches the predecessor. r_ab_n2 = _mm256_max_ps(r_ab_n2, v_eps_sq); __m256 inv_r_ab_n = _mm256_rsqrt_ps(r_ab_n2); __m256 r_ab_n = _mm256_mul_ps(r_ab_n2, inv_r_ab_n); __m256 r_ab2_x = _mm256_sub_ps(r_ab_x, sbe_jx); __m256 r_ab2_y = _mm256_sub_ps(r_ab_y, sbe_jy); __m256 r_ab2_n2 = _mm256_fmadd_ps(r_ab2_x, r_ab2_x, _mm256_mul_ps(r_ab2_y, r_ab2_y)); r_ab2_n2 = _mm256_max_ps(r_ab2_n2, v_eps_sq); __m256 inv_r_ab2_n = _mm256_rsqrt_ps(r_ab2_n2); __m256 r_ab2_n = _mm256_mul_ps(r_ab2_n2, inv_r_ab2_n); __m256 r_sum = _mm256_add_ps(r_ab_n, r_ab2_n); __m256 two_b_sq = _mm256_fmsub_ps(r_sum, r_sum, sb_jsq); two_b_sq = _mm256_max_ps(two_b_sq, v_eps_sq); __m256 inv_two_b = _mm256_rsqrt_ps(two_b_sq); __m256 two_b = _mm256_mul_ps(two_b_sq, inv_two_b); __m256 e_arg = _mm256_mul_ps(two_b, v_neg_2s_inv); e_arg = _mm256_max_ps(e_arg, v_exp_min); __m256 e_val = exp256_ps(e_arg); // smc = numer/denom in the predecessor. with inv-sqrts in hand the divide is free: // sm_r_ab = smc * r_ab2_n = numer / (two_b * r_ab_n) = numer * inv_two_b * inv_r_ab_n // sm_r_ab2 = smc * r_ab_n = numer / (two_b * r_ab2_n) = numer * inv_two_b * inv_r_ab2_n // for the i==j and padding lanes inv_two_b*inv_r_ab_n is huge, but the active-mask AND on // f_x/f_y below zeros the lane out before accumulation so it cant pollute anything. __m256 numer_inv_two_b = _mm256_mul_ps(_mm256_mul_ps(r_sum, v_neg_v0_div_2s), _mm256_mul_ps(e_val, inv_two_b)); __m256 sm_r_ab = _mm256_mul_ps(numer_inv_two_b, inv_r_ab_n); __m256 sm_r_ab2 = _mm256_mul_ps(numer_inv_two_b, inv_r_ab2_n); __m256 f_x = _mm256_fmadd_ps(r_ab2_x, sm_r_ab2, _mm256_mul_ps(r_ab_x, sm_r_ab)); __m256 f_y = _mm256_fmadd_ps(r_ab2_y, sm_r_ab2, _mm256_mul_ps(r_ab_y, sm_r_ab)); __m256 f_norm_sq = _mm256_fmadd_ps(f_x, f_x, _mm256_mul_ps(f_y, f_y)); __m256 rhs = _mm256_mul_ps(f_norm_sq, v_cos_phi_sq); __m256 e_dot_f = _mm256_fmadd_ps(eix, f_x, _mm256_mul_ps(eiy, f_y)); __m256 e_dot_f_sq = _mm256_mul_ps(e_dot_f, e_dot_f); __m256 c1 = _mm256_cmp_ps(e_dot_f, v_zero, _CMP_LT_OQ); __m256 c2 = _mm256_cmp_ps(e_dot_f_sq, rhs, _CMP_GT_OQ); __m256 cond = _mm256_and_ps(c1, c2); __m256 w_val = _mm256_blendv_ps(v_dt_neg, v_c_neg_dt, cond); __m256 f_x_safe = _mm256_and_ps(f_x, active); __m256 f_y_safe = _mm256_and_ps(f_y, active); acc_x = _mm256_fmadd_ps(f_x_safe, w_val, acc_x); acc_y = _mm256_fmadd_ps(f_y_safe, w_val, acc_y); } __m256 hsum = _mm256_hadd_ps(acc_x, acc_y); __m128 hsum_lo = _mm256_castps256_ps128(hsum); __m128 hsum_hi = _mm256_extractf128_ps(hsum, 1); __m128 sum4 = _mm_add_ps(hsum_lo, hsum_hi); sum4 = _mm_hadd_ps(sum4, sum4); alignas(16) float sum_out[4]; _mm_store_ps(sum_out, sum4); f_alpha_dt_x += sum_out[0]; f_alpha_dt_y += sum_out[1]; for (int k = 0; k < board.n_obstacles; k++) { float v1_x = pos_x[i] - board.obstacles[k].start_x; float v1_y = pos_y[i] - board.obstacles[k].start_y; float v2_x = board.obstacles[k].obstacle_v_x; float v2_y = board.obstacles[k].obstacle_v_y; float projection = (v1_x * v2_x + v1_y * v2_y) * board.obstacles[k].length_sq_inv; if (projection < 0.0) projection = 0.0; else if (projection > 1.0) projection = 1.0; float r_aB_x = pos_x[i] - (board.obstacles[k].start_x + board.obstacles[k].obstacle_v_x * projection); float r_aB_y = pos_y[i] - (board.obstacles[k].start_y + board.obstacles[k].obstacle_v_y * projection); float r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y); r_aB_norm = r_aB_norm > EPS ? r_aB_norm : EPS; float scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm; float f_alpha_B_neg_x = r_aB_x * scalar_mult; float f_alpha_B_neg_y = r_aB_y * scalar_mult; float w_val; float e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y[i] : e_y[i]); if (e_dot_f >= cos_phi) w_val = dt_neg; else w_val = c_neg_times_dt; float f_alpha_B_dt_x = f_alpha_B_neg_x * w_val; float f_alpha_B_dt_y = f_alpha_B_neg_y * w_val; f_alpha_dt_x += f_alpha_B_dt_x; f_alpha_dt_y += f_alpha_B_dt_y; } w_x[i] += f_alpha_dt_x; w_y[i] += f_alpha_dt_y; } for (int i = 0; i < n_p; i++) { float w_norm = sqrt(w_x[i] * w_x[i] + w_y[i] * w_y[i]); float w_norm2 = w_norm > EPS ? w_norm : EPS; float correction_factor = min(1.0f, v_max[i] / w_norm2); curr_vel_x[i] = w_x[i] * correction_factor; curr_vel_y[i] = w_y[i] * correction_factor; pos_x[i] = pos_x[i] + curr_vel_x[i] * dt; pos_y[i] = pos_y[i] + curr_vel_y[i] * dt; if (color[i] == 0 && pos_x[i] > (BOARD_LENGTH - 1.5)) { pos_x[i] = 1.0; destination_y[i] = pos_y[i]; w_x[i] = desired_v0[i]; w_y[i] = 0.0; curr_vel_x[i] = w_x[i]; curr_vel_y[i] = w_y[i]; } if (color[i] == 1 && pos_x[i] < 1.5) { pos_x[i] = BOARD_LENGTH - 1.0; destination_y[i] = pos_y[i]; w_x[i] = -desired_v0[i]; w_y[i] = 0.0; curr_vel_x[i] = w_x[i]; curr_vel_y[i] = w_y[i]; } pos_y[i] = clamp(pos_y[i], WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS); float ex = destination_x[i] - pos_x[i]; float ey = destination_y[i] - pos_y[i]; float e_norm = sqrt(ex * ex + ey * ey); float e_norm2 = e_norm > EPS ? e_norm : EPS; float e_norm_inv = 1.0 / e_norm2; e_x[i] = ex * e_norm_inv; e_y[i] = ey * e_norm_inv; float velocity_norm_sq = curr_vel_x[i] * curr_vel_x[i] + curr_vel_y[i] * curr_vel_y[i]; float velocity_norm = sqrt(velocity_norm_sq); sb_sq[i] = velocity_norm_sq * delta_t_sq; sb_e_x[i] = e_x[i] * (velocity_norm * delta_t); sb_e_y[i] = e_y[i] * (velocity_norm * delta_t); #if !BENCHMARK if (dump_path && (t % dump_interval == 0)) { samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]); } #endif } } #if !BENCHMARK if (dump_path) { ofstream out(dump_path); out << "time,id,x,y,p.e.x,p.e.y,color\n"; for (auto &[t, id, x, y, e_x, e_y, c] : samples) { out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n'; } } #endif return 0; }