Baseline Flop Count Update, New Float Baseline

scripts/verify.py

@@ -3,33 +3,37 @@ import csv
 import math
 
 if len(sys.argv) < 3:
-    sys.exit("Use it like: python verify.py reference.csv generated.csv [tol]")
-# We can use this later in case that we wanna compare something with a tolerance
-tol = float(sys.argv[3]) if len(sys.argv) > 3 else 0.00001
+  sys.exit('Use it like: python verify.py reference.csv generated.csv [tol]')
 
-with open(sys.argv[1], newline="") as f:
-    ref = list(csv.reader(f))
-with open(sys.argv[2], newline="") as f:
-    gen = list(csv.reader(f))
+TOL = float(sys.argv[3]) if len(sys.argv) > 3 else 1
+DIFF_COL_RATIO_THRESHOLD = 0.025
+
+with open(sys.argv[1], newline='') as f:
+  ref = list(csv.reader(f))
+with open(sys.argv[2], newline='') as f:
+  gen = list(csv.reader(f))
 
 if len(ref) != len(gen):
-    sys.exit(f"row count differs: ref={len(ref)} got={len(gen)}")
+  sys.exit(f'row count differs: ref={len(ref)} got={len(gen)}')
 
 header = ref[0]
+diff_col_count = 0
 for i in range(len(ref)):
-    if len(ref[i]) != len(gen[i]):
-        sys.exit(f"line {i+1}: column count differs")
-    for j in range(len(ref[i])):
-        a = ref[i][j].strip()
-        b = gen[i][j].strip()
-        if a == b:
-            continue
-        # in case that we wanna use tolerance
-        if tol > 0:
-            try:
-                if math.isclose(float(a), float(b), abs_tol=tol):
-                    continue
-            except ValueError:
-                pass
-        col = header[j] if j < len(header) else f"col{j}"
-        sys.exit(f"mismatch at line {i+1}, col {j} ('{col}'): ref='{a}' got='{b}'")
+  if len(ref[i]) != len(gen[i]):
+    sys.exit(f'line {i+1}: column count differs')
+  for j in range(len(ref[i])):
+    a = ref[i][j].strip()
+    b = gen[i][j].strip()
+    if a == b:
+      continue
+    if TOL > 0:
+      try:
+        if math.isclose(float(a), float(b), abs_tol=TOL):
+          continue
+        else:
+          diff_col_count += 1
+      except ValueError:
+        pass
+
+if diff_col_count / len(ref) > DIFF_COL_RATIO_THRESHOLD:
+  sys.exit(f'Error Threshold Exceeded!')