variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt_sr.cpp

This commit is contained in:
[Quy Anh] «Elliot» Nguyen
2026-05-25 14:01:10 +02:00
committed by Linus Baumberger
parent e7c63ea40a
commit 7448281515
5 changed files with 938 additions and 11 deletions
@@ -0,0 +1,476 @@
// # Changes: Performed Scalar Replacements
// # Predecessor: variant_inlining_8_SoA_AVX512_polyexp_fastinvsqrt.cpp
#include "../headers/board_reduced.hpp"
#include "../headers/constants.hpp"
#include "../headers/pedestrian_reduced.hpp"
#include <immintrin.h>
#include <fstream>
#include <random>
#include <vector>
#define SEED 42
#define BOARD_WIDTH 10.0
#define BOARD_LENGTH 50.0
#define WALL_EPS 1e-3
#define BOARD_WIDTH_MINUS_WALL_EPS 9.999
#ifndef RANDOM_SEED
#define RANDOM_SEED 0
#endif
#ifndef BENCHMARK
#define BENCHMARK 0
#endif
using namespace std;
alignas(64) vector<double> pos_x, pos_y;
alignas(64) vector<double> sb_sq;
alignas(64) vector<double> sb_e_x, sb_e_y;
alignas(64) vector<double> e_x, e_y;
alignas(64) vector<double> curr_vel_x, curr_vel_y;
alignas(64) vector<double> w_x, w_y;
alignas(64) vector<double> desired_v0;
alignas(64) vector<double> v_max;
alignas(64) vector<double> destination_x, destination_y;
alignas(64) vector<int> color;
Pedestrian *init_pedestrians(int n_p) {
#if RANDOM_SEED
random_device rd;
mt19937 gen(rd());
#else
mt19937 gen(SEED);
#endif
normal_distribution<double> velocity_gen(v0_mean, v0_std_dev);
uniform_real_distribution<double> y_gen(0.3, BOARD_WIDTH - 0.3);
uniform_real_distribution<double> x_gen_left(2.0, BOARD_LENGTH / 2.0);
uniform_real_distribution<double> x_gen_right(BOARD_LENGTH / 2.0, BOARD_LENGTH - 2.0);
const int n_p_pad = (n_p + 7) & ~7;
pos_x.resize(n_p_pad);
pos_y.resize(n_p_pad);
sb_sq.resize(n_p_pad);
sb_e_x.resize(n_p_pad);
sb_e_y.resize(n_p_pad);
e_x.resize(n_p_pad);
e_y.resize(n_p_pad);
curr_vel_x.resize(n_p_pad);
curr_vel_y.resize(n_p_pad);
w_x.resize(n_p_pad);
w_y.resize(n_p_pad);
desired_v0.resize(n_p_pad);
v_max.resize(n_p_pad);
destination_x.resize(n_p_pad);
destination_y.resize(n_p_pad);
color.resize(n_p_pad);
int n_p_half = n_p / 2;
for (int i = 0; i < n_p_half; i++) {
double x = x_gen_left(gen);
double y = y_gen(gen);
double dest_x = BOARD_LENGTH + 30.0;
double dest_y = y;
double init_e_x = dest_x - x;
double init_e_y = dest_y - y;
double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y);
double init_e_unit_x = init_e_x / init_e_norm;
double init_e_unit_y = init_e_y / init_e_norm;
double d_v0;
do {
d_v0 = velocity_gen(gen);
} while (d_v0 <= 0.1);
double v_m = 1.3 * d_v0;
color[i] = 0;
pos_x[i] = x;
pos_y[i] = y;
e_x[i] = init_e_unit_x;
e_y[i] = init_e_unit_y;
v_max[i] = v_m;
desired_v0[i] = d_v0;
destination_x[i] = dest_x;
destination_y[i] = dest_y;
w_x[i] = 0.0;
w_y[i] = 0.0;
curr_vel_x[i] = 0.0;
curr_vel_y[i] = 0.0;
sb_sq[i] = 0.0;
sb_e_x[i] = 0.0;
sb_e_y[i] = 0.0;
}
for (int i = n_p_half; i < n_p; i++) {
double x = x_gen_right(gen);
double y = y_gen(gen);
double dest_x = -30;
double dest_y = y;
double init_e_x = dest_x - x;
double init_e_y = dest_y - y;
double init_e_norm = sqrt(init_e_x * init_e_x + init_e_y * init_e_y);
double init_e_unit_x = init_e_x / init_e_norm;
double init_e_unit_y = init_e_y / init_e_norm;
double d_v0;
do {
d_v0 = velocity_gen(gen);
} while (d_v0 <= 0.1);
double v_m = 1.3 * d_v0;
color[i] = 1;
pos_x[i] = x;
pos_y[i] = y;
e_x[i] = init_e_unit_x;
e_y[i] = init_e_unit_y;
v_max[i] = v_m;
desired_v0[i] = d_v0;
destination_x[i] = dest_x;
destination_y[i] = dest_y;
w_x[i] = 0.0;
w_y[i] = 0.0;
curr_vel_x[i] = 0.0;
curr_vel_y[i] = 0.0;
sb_sq[i] = 0.0;
sb_e_x[i] = 0.0;
sb_e_y[i] = 0.0;
}
for (int j = n_p; j < n_p_pad; j++) {
pos_x[j] = 1e9;
pos_y[j] = 1e9;
sb_sq[j] = 0.0;
sb_e_x[j] = 0.0;
sb_e_y[j] = 0.0;
}
return nullptr;
}
Board init_board_without_obstacles() { return Board(BOARD_LENGTH, BOARD_WIDTH, 0, nullptr); }
static inline __m512d _avx512_exp_pd(__m512d x) {
const __m512d log2e = _mm512_set1_pd(1.4426950408889634);
__m512d nf = _mm512_roundscale_pd(_mm512_mul_pd(x, log2e), _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
const __m512d ln2 = _mm512_set1_pd(0.6931471805599453);
__m512d r = _mm512_fnmadd_pd(nf, ln2, x);
const __m512d c7 = _mm512_set1_pd(1.0 / 5040.0);
const __m512d c6 = _mm512_set1_pd(1.0 / 720.0);
const __m512d c5 = _mm512_set1_pd(1.0 / 120.0);
const __m512d c4 = _mm512_set1_pd(1.0 / 24.0);
const __m512d c3 = _mm512_set1_pd(1.0 / 6.0);
const __m512d c2 = _mm512_set1_pd(0.5);
const __m512d c1 = _mm512_set1_pd(1.0);
const __m512d c0 = _mm512_set1_pd(1.0);
__m512d p = c7;
p = _mm512_fmadd_pd(p, r, c6);
p = _mm512_fmadd_pd(p, r, c5);
p = _mm512_fmadd_pd(p, r, c4);
p = _mm512_fmadd_pd(p, r, c3);
p = _mm512_fmadd_pd(p, r, c2);
p = _mm512_fmadd_pd(p, r, c1);
p = _mm512_fmadd_pd(p, r, c0);
__m256d nf_lo = _mm512_castpd512_pd256(nf);
__m256d nf_hi = _mm512_extractf64x4_pd(nf, 1);
__m128i ni32_lo = _mm256_cvtpd_epi32(nf_lo);
__m128i ni32_hi = _mm256_cvtpd_epi32(nf_hi);
__m256i ni64_lo = _mm256_cvtepi32_epi64(ni32_lo);
__m256i ni64_hi = _mm256_cvtepi32_epi64(ni32_hi);
__m512i ni64 = _mm512_inserti64x4(_mm512_castsi256_si512(ni64_lo), ni64_hi, 1);
__m512i biased = _mm512_add_epi64(ni64, _mm512_set1_epi64(1023));
__m512i scaled = _mm512_slli_epi64(biased, 52);
__m512d pow2n = _mm512_castsi512_pd(scaled);
return _mm512_mul_pd(p, pow2n);
}
static inline double fast_inv_sqrt(double number) {
double y = number;
uint64_t i = bit_cast<uint64_t>(y);
i = 0x5fe6eb50c7b537a9ULL - (i >> 1);
y = bit_cast<double>(i);
double x2 = number * 0.5;
y = y * (1.5 - (x2 * y * y));
y = y * (1.5 - (x2 * y * y));
y = y * (1.5 - (x2 * y * y));
return y;
}
int run_simulation(Pedestrian *pedestrians, Board &board, int n_p, int n_iter, const char *dump_path) {
#if BENCHMARK
(void)dump_path;
#endif
#if !BENCHMARK
const int dump_interval = 1;
vector<tuple<int, int, double, double, double, double, int>> samples;
if (dump_path) {
samples.reserve(static_cast<size_t>(n_iter / dump_interval + 1) * n_p);
for (int i = 0; i < n_p; i++) {
samples.emplace_back(0, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]);
}
}
#endif
const int n_p_pad = (n_p + 7) & ~7;
alignas(64) constexpr double lane_offset_arr[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0};
const __m512d lane_off = _mm512_load_pd(lane_offset_arr);
const __m512d v_neg_v0_div_2s = _mm512_set1_pd(NEG_V0_AB_div_TWO_SIGMA);
const __m512d v_neg_2s_inv = _mm512_set1_pd(NEG_TWO_SIGMA_INV);
const __m512d v_cos_phi_inv = _mm512_set1_pd(cos_phi_inv);
const __m512d v_dt_neg = _mm512_set1_pd(dt_neg);
const __m512d v_c_neg_dt = _mm512_set1_pd(c_neg_times_dt);
const __m512d v_zero = _mm512_setzero_pd();
const __m512d v_np = _mm512_set1_pd(n_p);
for (int t = 0; t < n_iter; t++) {
for (int i = 0; i < n_p; i++) {
double pos_x_i = pos_x[i];
double pos_y_i = pos_y[i];
double e_x_i = e_x[i];
double e_y_i = e_y[i];
double desired_v0_i = desired_v0[i];
double curr_vel_x_i = curr_vel_x[i];
double curr_vel_y_i = curr_vel_y[i];
double f_alpha_dt_x = (e_x_i * desired_v0_i - curr_vel_x_i) * dt_times_tau_alpha_inv;
double f_alpha_dt_y = (e_y_i * desired_v0_i - curr_vel_y_i) * dt_times_tau_alpha_inv;
const __m512d pix = _mm512_set1_pd(pos_x_i);
const __m512d piy = _mm512_set1_pd(pos_y_i);
const __m512d eix = _mm512_set1_pd(e_x_i);
const __m512d eiy = _mm512_set1_pd(e_y_i);
const __m512d v_i = _mm512_set1_pd((double)i);
__m512d acc_x = _mm512_setzero_pd();
__m512d acc_y = _mm512_setzero_pd();
for (int j = 0; j < n_p_pad; j += 8) {
__m512d j_idx = _mm512_add_pd(_mm512_set1_pd((double)j), lane_off);
__mmask8 m_neq_i = _mm512_cmp_pd_mask(j_idx, v_i, _CMP_NEQ_OQ);
__mmask8 m_in_range = _mm512_cmp_pd_mask(j_idx, v_np, _CMP_LT_OQ);
__mmask8 active = m_neq_i & m_in_range;
__m512d pjx = _mm512_load_pd(&pos_x[j]);
__m512d pjy = _mm512_load_pd(&pos_y[j]);
__m512d sbe_jx = _mm512_load_pd(&sb_e_x[j]);
__m512d sbe_jy = _mm512_load_pd(&sb_e_y[j]);
__m512d sb_jsq = _mm512_load_pd(&sb_sq[j]);
__m512d r_ab_x = _mm512_sub_pd(pix, pjx);
__m512d r_ab_y = _mm512_sub_pd(piy, pjy);
__m512d r_ab_n2 = _mm512_fmadd_pd(r_ab_x, r_ab_x, _mm512_mul_pd(r_ab_y, r_ab_y));
__m512d r_ab_n = _mm512_sqrt_pd(r_ab_n2);
__m512d r_ab2_x = _mm512_sub_pd(r_ab_x, sbe_jx);
__m512d r_ab2_y = _mm512_sub_pd(r_ab_y, sbe_jy);
__m512d r_ab2_n2 = _mm512_fmadd_pd(r_ab2_x, r_ab2_x, _mm512_mul_pd(r_ab2_y, r_ab2_y));
__m512d r_ab2_n = _mm512_sqrt_pd(r_ab2_n2);
__m512d r_sum = _mm512_add_pd(r_ab_n, r_ab2_n);
__m512d two_b_sq = _mm512_fmsub_pd(r_sum, r_sum, sb_jsq);
__m512d two_b = _mm512_sqrt_pd(two_b_sq);
__m512d e_arg = _mm512_mul_pd(two_b, v_neg_2s_inv);
__m512d e_val = _avx512_exp_pd(e_arg);
__m512d numer = _mm512_mul_pd(_mm512_mul_pd(r_sum, v_neg_v0_div_2s), e_val);
__m512d denom = _mm512_mul_pd(_mm512_mul_pd(two_b, r_ab_n), r_ab2_n);
__m512d denom_safe = _mm512_mask_blend_pd(active, _mm512_set1_pd(1.0), denom);
__m512d smc = _mm512_div_pd(numer, denom_safe);
__m512d sm_r_ab = _mm512_mul_pd(smc, r_ab2_n);
__m512d sm_r_ab2 = _mm512_mul_pd(smc, r_ab_n);
__m512d f_x = _mm512_fmadd_pd(r_ab2_x, sm_r_ab2, _mm512_mul_pd(r_ab_x, sm_r_ab));
__m512d f_y = _mm512_fmadd_pd(r_ab2_y, sm_r_ab2, _mm512_mul_pd(r_ab_y, sm_r_ab));
__m512d a = _mm512_mul_pd(_mm512_fmadd_pd(eix, f_x, _mm512_mul_pd(eiy, f_y)), v_cos_phi_inv);
__m512d a_sq = _mm512_mul_pd(a, a);
__m512d b_sq = _mm512_fmadd_pd(f_x, f_x, _mm512_mul_pd(f_y, f_y));
__mmask8 k_ge = _mm512_cmp_pd_mask(a, v_zero, _CMP_GE_OQ);
__mmask8 k_gt = _mm512_cmp_pd_mask(a_sq, b_sq, _CMP_GT_OQ);
__mmask8 k_cond = _mm512_kand(k_ge, k_gt);
__m512d w_val = _mm512_mask_blend_pd(k_cond, v_dt_neg, v_c_neg_dt);
__m512d f_x_safe = _mm512_maskz_mov_pd(active, f_x);
__m512d f_y_safe = _mm512_maskz_mov_pd(active, f_y);
acc_x = _mm512_fmadd_pd(f_x_safe, w_val, acc_x);
acc_y = _mm512_fmadd_pd(f_y_safe, w_val, acc_y);
}
f_alpha_dt_x += _mm512_reduce_add_pd(acc_x);
f_alpha_dt_y += _mm512_reduce_add_pd(acc_y);
for (int k = 0; k < board.n_obstacles; k++) {
double obs_start_x = board.obstacles[k].start_x;
double obs_start_y = board.obstacles[k].start_y;
double obs_v_x = board.obstacles[k].obstacle_v_x;
double obs_v_y = board.obstacles[k].obstacle_v_y;
double obs_length_sq_inv = board.obstacles[k].length_sq_inv;
double v1_x = pos_x_i - obs_start_x;
double v1_y = pos_y_i - obs_start_y;
double projection = (v1_x * obs_v_x + v1_y * obs_v_y) * obs_length_sq_inv;
if (projection < 0.0)
projection = 0.0;
else if (projection > 1.0)
projection = 1.0;
double r_aB_x = pos_x_i - (obs_start_x + obs_v_x * projection);
double r_aB_y = pos_y_i - (obs_start_y + obs_v_y * projection);
double r_aB_norm = sqrt(r_aB_x * r_aB_x + r_aB_y * r_aB_y);
double scalar_mult = (NEG_U0_AB_div_R * exp(r_aB_norm * NEG_R_INV)) / r_aB_norm;
double f_alpha_B_neg_x = r_aB_x * scalar_mult;
double f_alpha_B_neg_y = r_aB_y * scalar_mult;
double w_val = dt_neg;
double e_dot_f = (f_alpha_B_neg_y == 0) ? 0 : (f_alpha_B_neg_y < 0 ? -e_y_i : e_y_i);
if (e_dot_f < cos_phi)
w_val = c_neg_times_dt;
f_alpha_dt_x += f_alpha_B_neg_x * w_val;
f_alpha_dt_y += f_alpha_B_neg_y * w_val;
}
w_x[i] += f_alpha_dt_x;
w_y[i] += f_alpha_dt_y;
}
for (int i = 0; i < n_p; i++) {
double color_i = color[i];
double curr_vel_x_i = curr_vel_x[i];
double curr_vel_y_i = curr_vel_y[i];
double w_x_i = w_x[i];
double w_y_i = w_y[i];
double pos_x_i = pos_x[i];
double pos_y_i = pos_y[i];
double e_x_i = e_x[i];
double e_y_i = e_y[i];
double v_max_i = v_max[i];
double desired_v0_i = desired_v0[i];
double destination_x_i = destination_x[i];
double destination_y_i = destination_y[i];
double w_norm_inv = fast_inv_sqrt(w_x_i * w_x_i + w_y_i * w_y_i);
double correction_factor = min(1.0, v_max_i * w_norm_inv);
curr_vel_x_i = w_x_i * correction_factor;
curr_vel_y_i = w_y_i * correction_factor;
pos_x_i = pos_x_i + curr_vel_x_i * dt;
pos_y_i = pos_y_i + curr_vel_y_i * dt;
if (!color_i && pos_x_i > (BOARD_LENGTH - 1.5)) {
pos_x_i = 1.0;
destination_y_i = pos_y_i;
w_x_i = desired_v0_i;
w_y_i = 0.0;
curr_vel_x_i = w_x_i;
curr_vel_y_i = w_y_i;
}
if (color_i && pos_x_i < 1.5) {
pos_x_i = BOARD_LENGTH - 1.0;
destination_y_i = pos_y_i;
w_x_i = -desired_v0_i;
w_y_i = 0.0;
curr_vel_x_i = w_x_i;
curr_vel_y_i = w_y_i;
}
pos_y_i = clamp(pos_y_i, WALL_EPS, BOARD_WIDTH_MINUS_WALL_EPS);
double ex = destination_x_i - pos_x_i;
double ey = destination_y_i - pos_y_i;
double e_norm_inv = fast_inv_sqrt(ex * ex + ey * ey);
e_x_i = ex * e_norm_inv;
e_y_i = ey * e_norm_inv;
double velocity_norm_sq = curr_vel_x_i * curr_vel_x_i + curr_vel_y_i * curr_vel_y_i;
double sb_sq_i = velocity_norm_sq * delta_t_sq;
double velocity_norm_mul_delta_t = sqrt(sb_sq_i);
curr_vel_x[i] = curr_vel_x_i;
curr_vel_y[i] = curr_vel_y_i;
w_x[i] = w_x_i;
w_y[i] = w_y_i;
pos_x[i] = pos_x_i;
pos_y[i] = pos_y_i;
e_x[i] = e_x_i;
e_y[i] = e_y_i;
destination_y[i] = destination_y_i;
sb_sq[i] = sb_sq_i;
sb_e_x[i] = e_x_i * velocity_norm_mul_delta_t;
sb_e_y[i] = e_y_i * velocity_norm_mul_delta_t;
#if !BENCHMARK
if (dump_path && (t % dump_interval == 0)) {
samples.emplace_back(t + 1, i, pos_x[i], pos_y[i], e_x[i], e_y[i], color[i]);
}
#endif
}
}
#if !BENCHMARK
if (dump_path) {
ofstream out(dump_path);
out << "time,id,x,y,p.e.x,p.e.y,color\n";
for (auto &[t, id, x, y, e_x, e_y, c] : samples) {
out << t << ',' << id << ',' << x << ',' << y << ',' << e_x << ',' << e_y << ',' << c << '\n';
}
}
#endif
return 0;
}