diff --git a/Makefile b/Makefile index a062ea8..fb1f830 100644 --- a/Makefile +++ b/Makefile @@ -101,8 +101,8 @@ run: main # make verify VARIANT=v1_cutoff # check a specific variant verify: main ./$(TARGET) $(REFERENCE_N_P) $(REFERENCE_N_ITER) $(VERIFY_CSV) - @if cmp -s $(REFERENCE_CSV) $(VERIFY_CSV); then \ - echo "PASS: variant '$(VARIANT)' matches $(REFERENCE_CSV) bit-exact"; \ + @if python3 verify.py $(REFERENCE_CSV) $(VERIFY_CSV); then \ + echo "PASS: variant '$(VARIANT)' matches $(REFERENCE_CSV)"; \ rm -f $(VERIFY_CSV); \ else \ echo "FAIL: variant '$(VARIANT)' differs from $(REFERENCE_CSV) (kept $(VERIFY_CSV) for inspection)"; \ diff --git a/README.md b/README.md index 7e04a2a..45c2d1c 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,16 @@ make test Compiles and runs all unit tests. +```bash +make verify +``` + +Runs the simulation with the reference parameters and compares the output to `reference/baseline_n60_i4000.csv` using `verify.py`. Useful to check that a variant gives the same result as the baseline: + +```bash +make verify VARIANT=v1_cutoff +``` + ```bash make asm ``` @@ -356,12 +366,12 @@ make test VARIANT=v1_cutoff make run-bench VARIANT=v1_cutoff N_P=100 N_ITER=10000 ``` ### Verifying correctness -A reference CSV is stored in reference/baseline_n60_i4000.csv. To verify that a variant produces the same simulation output: +A reference CSV is stored in `reference/baseline_n60_i4000.csv`. To check that a variant produces the same simulation output, run: + ```bash -make run VARIANT=v1_cutoff N_P=60 N_ITER=4000 CSV=test_output.csv -diff reference/baseline_n60_i4000.csv test_output.csv +make verify VARIANT=v1_cutoff ``` -If the variant changes numerical results (e.g., distance cutoff skipping negligible forces), use a tolerance-based comparison instead of exact diff and justify the deviation. +If the variant changes numerical results (e.g., distance cutoff skipping negligible forces), use a tolerance-based comparison (use the python script with a third argument (tolerance)) instead of exact diff and justify the deviation. --- diff --git a/verify.py b/verify.py new file mode 100644 index 0000000..64126f4 --- /dev/null +++ b/verify.py @@ -0,0 +1,35 @@ +import sys +import csv +import math + +if len(sys.argv) < 3: + sys.exit("Use it like: python verify.py reference.csv generated.csv [tol]") +# We can use this later in case that we wanna compare something with a tolerance +tol = float(sys.argv[3]) if len(sys.argv) > 3 else 0.0 + +with open(sys.argv[1], newline="") as f: + ref = list(csv.reader(f)) +with open(sys.argv[2], newline="") as f: + gen = list(csv.reader(f)) + +if len(ref) != len(gen): + sys.exit(f"row count differs: ref={len(ref)} got={len(gen)}") + +header = ref[0] +for i in range(len(ref)): + if len(ref[i]) != len(gen[i]): + sys.exit(f"line {i+1}: column count differs") + for j in range(len(ref[i])): + a = ref[i][j].strip() + b = gen[i][j].strip() + if a == b: + continue + # in case that we wanna use tolerance + if tol > 0: + try: + if math.isclose(float(a), float(b), abs_tol=tol): + continue + except ValueError: + pass + col = header[j] if j < len(header) else f"col{j}" + sys.exit(f"mismatch at line {i+1}, col {j} ('{col}'): ref='{a}' got='{b}'")