Merge branch 'linus/new_makefile_command_to_verify' into 'main'

added a verify command to our makefile. so for future variants we can just use...

See merge request course-asl-fastcode26/team70!12

Makefile

@@ -101,8 +101,8 @@ run: main
 #   make verify VARIANT=v1_cutoff   # check a specific variant
 verify: main
 	./$(TARGET) $(REFERENCE_N_P) $(REFERENCE_N_ITER) $(VERIFY_CSV)
-	@if cmp -s $(REFERENCE_CSV) $(VERIFY_CSV); then \
+	@if  python3 verify.py $(REFERENCE_CSV) $(VERIFY_CSV); then \
-		echo "PASS: variant '$(VARIANT)' matches $(REFERENCE_CSV) bit-exact"; \
+		echo "PASS: variant '$(VARIANT)' matches $(REFERENCE_CSV)"; \
 		rm -f $(VERIFY_CSV); \
 	else \
 		echo "FAIL: variant '$(VARIANT)' differs from $(REFERENCE_CSV) (kept $(VERIFY_CSV) for inspection)"; \

README.md

@@ -108,6 +108,16 @@ make test
 
 Compiles and runs all unit tests.
 
+```bash
+make verify
+```
+
+Runs the simulation with the reference parameters and compares the output to `reference/baseline_n60_i4000.csv` using `verify.py`. Useful to check that a variant gives the same result as the baseline:
+
+```bash
+make verify VARIANT=v1_cutoff
+```
+
 ```bash
 make asm
 ```
@@ -356,12 +366,12 @@ make test VARIANT=v1_cutoff
 make run-bench VARIANT=v1_cutoff N_P=100 N_ITER=10000
 ```
 ### Verifying correctness
-A reference CSV is stored in reference/baseline_n60_i4000.csv. To verify that a variant produces the same simulation output:
+A reference CSV is stored in `reference/baseline_n60_i4000.csv`. To check that a variant produces the same simulation output, run:
+
 ```bash
-make run VARIANT=v1_cutoff N_P=60 N_ITER=4000 CSV=test_output.csv
+make verify VARIANT=v1_cutoff
-diff reference/baseline_n60_i4000.csv test_output.csv
 ```
-If the variant changes numerical results (e.g., distance cutoff skipping negligible forces), use a tolerance-based comparison instead of exact diff and justify the deviation.
+If the variant changes numerical results (e.g., distance cutoff skipping negligible forces), use a tolerance-based comparison (use the python script with a third argument (tolerance)) instead of exact diff and justify the deviation.
 
 
 ---

verify.py

@@ -0,0 +1,35 @@
+import sys
+import csv
+import math
+
+if len(sys.argv) < 3:
+    sys.exit("Use it like: python verify.py reference.csv generated.csv [tol]")
+# We can use this later in case that we wanna compare something with a tolerance
+tol = float(sys.argv[3]) if len(sys.argv) > 3 else 0.0
+
+with open(sys.argv[1], newline="") as f:
+    ref = list(csv.reader(f))
+with open(sys.argv[2], newline="") as f:
+    gen = list(csv.reader(f))
+
+if len(ref) != len(gen):
+    sys.exit(f"row count differs: ref={len(ref)} got={len(gen)}")
+
+header = ref[0]
+for i in range(len(ref)):
+    if len(ref[i]) != len(gen[i]):
+        sys.exit(f"line {i+1}: column count differs")
+    for j in range(len(ref[i])):
+        a = ref[i][j].strip()
+        b = gen[i][j].strip()
+        if a == b:
+            continue
+        # in case that we wanna use tolerance
+        if tol > 0:
+            try:
+                if math.isclose(float(a), float(b), abs_tol=tol):
+                    continue
+            except ValueError:
+                pass
+        col = header[j] if j < len(header) else f"col{j}"
+        sys.exit(f"mismatch at line {i+1}, col {j} ('{col}'): ref='{a}' got='{b}'")