Added in a t-test for the test but I got it off google AI summary so definitely need to confirm it. Looked alright at first glance though.

src/stats.cpp

@@ -1,5 +1,7 @@
 #include "stats.hpp"
 #include <fmt/core.h>
+#include <numeric>
+#include <numbers>
 
 void Stats::dump(std::string msg)
 {
@@ -8,3 +10,32 @@ void Stats::dump(std::string msg)
             msg, sum, sumsq, n, min, max, mean(),
             stddev());
 }
+
+// WARNING: got this code from google AI. PROBABLY BULLSHIT
+void Stats::t_test(Stats& other) {
+  double n1 = this->n;
+  double n2 = other.n;
+
+  double mean1 = this->mean();
+  double mean2 = other.mean();
+
+  double var1 = this->variance();
+  double var2 = other.variance();
+
+  // calculate t-stat
+  double delta_mean = mean1 - mean2;
+  double pooled_se = std::sqrt((var1 / n1) + (var2 / n2));
+  double t_stat = delta_mean / pooled_se;
+
+  // calculate degrees of freedom
+  double num = std::pow((var1 / n1) + (var2 / n2), 2);
+  double den = (std::pow(var1 / n1, 2) / (n1 - 1)) + (std::pow(var2 / n2, 2) / (n2 - 1));
+  double df = num / den;
+
+  double x = df / (df + t_stat * t_stat);
+
+  double p_val = std::tgamma((df + 1.0) / 2.0) / (std::sqrt(df * std::numbers::pi) * std::tgamma(df / 2.0)) * std::pow(1.0 + (t_stat * t_stat) / df, -(df + 1.0) / 2.0);
+
+  fmt::println("t_stat={}, df={}, p_val={}",
+      t_stat, df, p_val);
+}

src/stats.hpp

@@ -60,4 +60,6 @@ struct Stats {
   }
 
   void dump(std::string msg="");
+
+  void t_test(Stats& other);
 };

tests/perf_tests.cpp

@@ -139,7 +139,7 @@ namespace perf_tests {
     S5(int x) : a17(x), a18(x), a19(x), a20(x) { }
   };
 
-  struct MultipleSOA {
+  struct MultipleAOS {
     std::vector<S1> s1;
     std::vector<S2> s2;
     std::vector<S3> s3;
@@ -160,8 +160,8 @@ namespace perf_tests {
     std::vector<Stats> soa_many_for;
     std::vector<Stats> ls_big_for;
     std::vector<Stats> ls_many_for;
-    std::vector<Stats> moa_big_for;
+    std::vector<Stats> maos_big_for;
-    std::vector<Stats> moa_many_for;
+    std::vector<Stats> maos_many_for;
   };
 
   void test_large_soa(Samples& samples, int sample_size, int sample_number) {
@@ -290,125 +290,134 @@ namespace perf_tests {
   }
 
 
-  void test_moa_style(Samples& samples, int sample_size, int sample_number) {
+  void test_maos_style(Samples& samples, int sample_size, int sample_number) {
-    auto moa = new MultipleSOA();
+    auto maos = new MultipleAOS();
 
     for(int i = 0; i < sample_size; i++) {
-      moa->push_back(i);
+      maos->push_back(i);
     }
 
-    Stats moa_big_for;
+    Stats maos_big_for;
     for(int sample = 0; sample < sample_number; sample++) {
-      auto start = moa_big_for.time_start();
+      auto start = maos_big_for.time_start();
       for(int i = 0; i < sample_size; i++) {
-        moa->s1[i].a01 += moa->s1[i].a02 + sample;
+        maos->s1[i].a01 += maos->s1[i].a02 + sample;
-        moa->s1[i].a02 += moa->s1[i].a03 + sample;
+        maos->s1[i].a02 += maos->s1[i].a03 + sample;
-        moa->s1[i].a03 += moa->s1[i].a04 + sample;
+        maos->s1[i].a03 += maos->s1[i].a04 + sample;
-        moa->s1[i].a04 += moa->s2[i].a05 + sample;
+        maos->s1[i].a04 += maos->s2[i].a05 + sample;
-
+
-        moa->s2[i].a05 += moa->s2[i].a06 + sample;
+        maos->s2[i].a05 += maos->s2[i].a06 + sample;
-        moa->s2[i].a06 += moa->s2[i].a07 + sample;
+        maos->s2[i].a06 += maos->s2[i].a07 + sample;
-        moa->s2[i].a07 += moa->s2[i].a08 + sample;
+        maos->s2[i].a07 += maos->s2[i].a08 + sample;
-        moa->s2[i].a08 += moa->s3[i].a09 + sample;
+        maos->s2[i].a08 += maos->s3[i].a09 + sample;
-
+
-        moa->s3[i].a09 += moa->s3[i].a10 + sample;
+        maos->s3[i].a09 += maos->s3[i].a10 + sample;
-        moa->s3[i].a10 += moa->s3[i].a11 + sample;
+        maos->s3[i].a10 += maos->s3[i].a11 + sample;
-        moa->s3[i].a11 += moa->s3[i].a12 + sample;
+        maos->s3[i].a11 += maos->s3[i].a12 + sample;
-        moa->s3[i].a12 += moa->s4[i].a13 + sample;
+        maos->s3[i].a12 += maos->s4[i].a13 + sample;
-
+
-        moa->s4[i].a13 += moa->s4[i].a14 + sample;
+        maos->s4[i].a13 += maos->s4[i].a14 + sample;
-        moa->s4[i].a14 += moa->s4[i].a15 + sample;
+        maos->s4[i].a14 += maos->s4[i].a15 + sample;
-        moa->s4[i].a15 += moa->s4[i].a16 + sample;
+        maos->s4[i].a15 += maos->s4[i].a16 + sample;
-        moa->s4[i].a16 += moa->s5[i].a17 + sample;
+        maos->s4[i].a16 += maos->s5[i].a17 + sample;
-
+
-        moa->s5[i].a17 += moa->s5[i].a18 + sample;
+        maos->s5[i].a17 += maos->s5[i].a18 + sample;
-        moa->s5[i].a18 += moa->s5[i].a19 + sample;
+        maos->s5[i].a18 += maos->s5[i].a19 + sample;
-        moa->s5[i].a19 += moa->s5[i].a20 + sample;
+        maos->s5[i].a19 += maos->s5[i].a20 + sample;
       }
-      moa_big_for.sample_time(start);
+      maos_big_for.sample_time(start);
     }
-    samples.moa_big_for.push_back(moa_big_for);
+    samples.maos_big_for.push_back(maos_big_for);
 
-    Stats moa_many_for;
+    Stats maos_many_for;
     for(int sample = 0; sample < sample_number; sample++) {
-      auto start = moa_many_for.time_start();
+      auto start = maos_many_for.time_start();
 
-      for(auto& el : moa->s1) {
+      for(auto& el : maos->s1) {
         el.a01 += el.a02 + sample;
         el.a02 += el.a03 + sample;
         el.a03 += el.a04 + sample;
       }
 
       for(int i = 0; i < sample_size; i++) {
-        moa->s1[i].a04 += moa->s2[i].a05 + sample;
+        maos->s1[i].a04 += maos->s2[i].a05 + sample;
       }
 
-      for(auto& el : moa->s2) {
+      for(auto& el : maos->s2) {
         el.a05 += el.a06 + sample;
         el.a06 += el.a07 + sample;
         el.a07 += el.a08 + sample;
       }
 
       for(int i = 0; i < sample_size; i++) {
-        moa->s2[i].a08 += moa->s3[i].a09 + i;
+        maos->s2[i].a08 += maos->s3[i].a09 + i;
       }
 
-      for(auto& el : moa->s3) {
+      for(auto& el : maos->s3) {
         el.a09 += el.a10 + sample;
         el.a10 += el.a11 + sample;
         el.a11 += el.a12 + sample;
       }
 
       for(int i = 0; i < sample_size; i++) {
-        moa->s3[i].a12 += moa->s4[i].a13 + sample;
+        maos->s3[i].a12 += maos->s4[i].a13 + sample;
       }
 
-      for(auto& el : moa->s4) {
+      for(auto& el : maos->s4) {
         el.a13 += el.a14 + sample;
         el.a14 += el.a15 + sample;
         el.a15 += el.a16 + sample;
       }
 
       for(int i = 0; i < sample_size; i++) {
-        moa->s4[i].a16 += moa->s5[i].a17 + sample;
+        maos->s4[i].a16 += maos->s5[i].a17 + sample;
       }
 
-      for(auto& el : moa->s5) {
+      for(auto& el : maos->s5) {
         el.a17 += el.a18 + sample;
         el.a18 += el.a19 + sample;
         el.a19 += el.a20 + sample;
       }
 
-      moa_many_for.sample_time(start);
+      maos_many_for.sample_time(start);
     }
 
-    samples.moa_many_for.push_back(moa_many_for);
+    samples.maos_many_for.push_back(maos_many_for);
   }
 
-  void test_soa_performance() {
+  void test_all_performance() {
     Samples samples;
 
-    int sample_size = 1000;
+    int sample_size = 100;
-    int sample_count = 1000;
+    int sample_count = 100;
 
     for(int i = 0; i < 100; i++) {
       // quick and dirty "randomness"
       if(i % 2 == 0) {
         test_large_soa(samples, sample_size, sample_count);
         test_large_struct(samples, sample_size, sample_count);
-        test_moa_style(samples, sample_count, sample_count);
+        test_maos_style(samples, sample_count, sample_count);
       } else if(i % 3 == 0) {
         test_large_struct(samples, sample_size, sample_count);
         test_large_soa(samples, sample_size, sample_count);
-        test_moa_style(samples, sample_count, sample_count);
+        test_maos_style(samples, sample_count, sample_count);
       } else {
-        test_moa_style(samples, sample_count, sample_count);
+        test_maos_style(samples, sample_count, sample_count);
         test_large_struct(samples, sample_size, sample_count);
         test_large_soa(samples, sample_size, sample_count);
       }
     }
 
-    fmt::println("soa_bf soa_mf ls_bf ls_mf moa_bf moa_mf");
+    fmt::println("soa_bf soa_mf ls_bf ls_mf maos_bf maos_mf");
+
+    Samples mofm;
+
+    Stats soa_big_for;
+    Stats soa_many_for;
+    Stats ls_big_for;
+    Stats ls_many_for;
+    Stats maos_big_for;
+    Stats maos_many_for;
 
     for(size_t i = 0; i < samples.soa_big_for.size(); i++) {
       fmt::println("{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}",
@@ -416,15 +425,37 @@ namespace perf_tests {
           samples.soa_many_for[i].mean(),
           samples.ls_big_for[i].mean(),
           samples.ls_many_for[i].mean(),
-          samples.moa_big_for[i].mean(),
+          samples.maos_big_for[i].mean(),
-          samples.moa_many_for[i].mean());
+          samples.maos_many_for[i].mean());
+
+      // this is potentially total bullshit
+      soa_big_for.sample(samples.soa_big_for[i].mean());
+      soa_many_for.sample(samples.soa_many_for[i].mean());
+      ls_big_for.sample(samples.ls_big_for[i].mean());
+      ls_many_for.sample(samples.ls_many_for[i].mean());
+      maos_big_for.sample(samples.maos_big_for[i].mean());
+      maos_many_for.sample(samples.maos_many_for[i].mean());
     }
+
+    // totally not sure if this t-test is legit, must study more
+    soa_big_for.t_test(ls_big_for);
+    soa_big_for.t_test(maos_big_for);
+
+    soa_many_for.t_test(ls_many_for);
+    soa_many_for.t_test(maos_many_for);
+
+    ls_many_for.t_test(maos_many_for);
+    ls_big_for.t_test(maos_big_for);
+
+    // the most pathologic case
+    fmt::println("---- {}", soa_big_for.mean() - ls_many_for.mean());
+    soa_big_for.t_test(ls_many_for);
   }
 
   fuc2::Set TESTS{
     .name="perf",
       .tests={
-        TEST(test_soa_performance),
+        TEST(test_all_performance),
       }
   };
 }