diff --git a/src/stats.cpp b/src/stats.cpp index 5d26c83..3bd0039 100644 --- a/src/stats.cpp +++ b/src/stats.cpp @@ -1,5 +1,7 @@ #include "stats.hpp" #include +#include +#include void Stats::dump(std::string msg) { @@ -8,3 +10,32 @@ void Stats::dump(std::string msg) msg, sum, sumsq, n, min, max, mean(), stddev()); } + +// WARNING: got this code from google AI. PROBABLY BULLSHIT +void Stats::t_test(Stats& other) { + double n1 = this->n; + double n2 = other.n; + + double mean1 = this->mean(); + double mean2 = other.mean(); + + double var1 = this->variance(); + double var2 = other.variance(); + + // calculate t-stat + double delta_mean = mean1 - mean2; + double pooled_se = std::sqrt((var1 / n1) + (var2 / n2)); + double t_stat = delta_mean / pooled_se; + + // calculate degrees of freedom + double num = std::pow((var1 / n1) + (var2 / n2), 2); + double den = (std::pow(var1 / n1, 2) / (n1 - 1)) + (std::pow(var2 / n2, 2) / (n2 - 1)); + double df = num / den; + + double x = df / (df + t_stat * t_stat); + + double p_val = std::tgamma((df + 1.0) / 2.0) / (std::sqrt(df * std::numbers::pi) * std::tgamma(df / 2.0)) * std::pow(1.0 + (t_stat * t_stat) / df, -(df + 1.0) / 2.0); + + fmt::println("t_stat={}, df={}, p_val={}", + t_stat, df, p_val); +} diff --git a/src/stats.hpp b/src/stats.hpp index ef7d717..5d819a4 100644 --- a/src/stats.hpp +++ b/src/stats.hpp @@ -60,4 +60,6 @@ struct Stats { } void dump(std::string msg=""); + + void t_test(Stats& other); }; diff --git a/tests/perf_tests.cpp b/tests/perf_tests.cpp index 2c708b2..ceb5f3a 100644 --- a/tests/perf_tests.cpp +++ b/tests/perf_tests.cpp @@ -139,7 +139,7 @@ namespace perf_tests { S5(int x) : a17(x), a18(x), a19(x), a20(x) { } }; - struct MultipleSOA { + struct MultipleAOS { std::vector s1; std::vector s2; std::vector s3; @@ -160,8 +160,8 @@ namespace perf_tests { std::vector soa_many_for; std::vector ls_big_for; std::vector ls_many_for; - std::vector moa_big_for; - std::vector moa_many_for; + std::vector maos_big_for; + std::vector maos_many_for; }; void test_large_soa(Samples& samples, int sample_size, int sample_number) { @@ -290,125 +290,134 @@ namespace perf_tests { } - void test_moa_style(Samples& samples, int sample_size, int sample_number) { - auto moa = new MultipleSOA(); + void test_maos_style(Samples& samples, int sample_size, int sample_number) { + auto maos = new MultipleAOS(); for(int i = 0; i < sample_size; i++) { - moa->push_back(i); + maos->push_back(i); } - Stats moa_big_for; + Stats maos_big_for; for(int sample = 0; sample < sample_number; sample++) { - auto start = moa_big_for.time_start(); + auto start = maos_big_for.time_start(); for(int i = 0; i < sample_size; i++) { - moa->s1[i].a01 += moa->s1[i].a02 + sample; - moa->s1[i].a02 += moa->s1[i].a03 + sample; - moa->s1[i].a03 += moa->s1[i].a04 + sample; - moa->s1[i].a04 += moa->s2[i].a05 + sample; - - moa->s2[i].a05 += moa->s2[i].a06 + sample; - moa->s2[i].a06 += moa->s2[i].a07 + sample; - moa->s2[i].a07 += moa->s2[i].a08 + sample; - moa->s2[i].a08 += moa->s3[i].a09 + sample; - - moa->s3[i].a09 += moa->s3[i].a10 + sample; - moa->s3[i].a10 += moa->s3[i].a11 + sample; - moa->s3[i].a11 += moa->s3[i].a12 + sample; - moa->s3[i].a12 += moa->s4[i].a13 + sample; - - moa->s4[i].a13 += moa->s4[i].a14 + sample; - moa->s4[i].a14 += moa->s4[i].a15 + sample; - moa->s4[i].a15 += moa->s4[i].a16 + sample; - moa->s4[i].a16 += moa->s5[i].a17 + sample; - - moa->s5[i].a17 += moa->s5[i].a18 + sample; - moa->s5[i].a18 += moa->s5[i].a19 + sample; - moa->s5[i].a19 += moa->s5[i].a20 + sample; + maos->s1[i].a01 += maos->s1[i].a02 + sample; + maos->s1[i].a02 += maos->s1[i].a03 + sample; + maos->s1[i].a03 += maos->s1[i].a04 + sample; + maos->s1[i].a04 += maos->s2[i].a05 + sample; + + maos->s2[i].a05 += maos->s2[i].a06 + sample; + maos->s2[i].a06 += maos->s2[i].a07 + sample; + maos->s2[i].a07 += maos->s2[i].a08 + sample; + maos->s2[i].a08 += maos->s3[i].a09 + sample; + + maos->s3[i].a09 += maos->s3[i].a10 + sample; + maos->s3[i].a10 += maos->s3[i].a11 + sample; + maos->s3[i].a11 += maos->s3[i].a12 + sample; + maos->s3[i].a12 += maos->s4[i].a13 + sample; + + maos->s4[i].a13 += maos->s4[i].a14 + sample; + maos->s4[i].a14 += maos->s4[i].a15 + sample; + maos->s4[i].a15 += maos->s4[i].a16 + sample; + maos->s4[i].a16 += maos->s5[i].a17 + sample; + + maos->s5[i].a17 += maos->s5[i].a18 + sample; + maos->s5[i].a18 += maos->s5[i].a19 + sample; + maos->s5[i].a19 += maos->s5[i].a20 + sample; } - moa_big_for.sample_time(start); + maos_big_for.sample_time(start); } - samples.moa_big_for.push_back(moa_big_for); + samples.maos_big_for.push_back(maos_big_for); - Stats moa_many_for; + Stats maos_many_for; for(int sample = 0; sample < sample_number; sample++) { - auto start = moa_many_for.time_start(); + auto start = maos_many_for.time_start(); - for(auto& el : moa->s1) { + for(auto& el : maos->s1) { el.a01 += el.a02 + sample; el.a02 += el.a03 + sample; el.a03 += el.a04 + sample; } for(int i = 0; i < sample_size; i++) { - moa->s1[i].a04 += moa->s2[i].a05 + sample; + maos->s1[i].a04 += maos->s2[i].a05 + sample; } - for(auto& el : moa->s2) { + for(auto& el : maos->s2) { el.a05 += el.a06 + sample; el.a06 += el.a07 + sample; el.a07 += el.a08 + sample; } for(int i = 0; i < sample_size; i++) { - moa->s2[i].a08 += moa->s3[i].a09 + i; + maos->s2[i].a08 += maos->s3[i].a09 + i; } - for(auto& el : moa->s3) { + for(auto& el : maos->s3) { el.a09 += el.a10 + sample; el.a10 += el.a11 + sample; el.a11 += el.a12 + sample; } for(int i = 0; i < sample_size; i++) { - moa->s3[i].a12 += moa->s4[i].a13 + sample; + maos->s3[i].a12 += maos->s4[i].a13 + sample; } - for(auto& el : moa->s4) { + for(auto& el : maos->s4) { el.a13 += el.a14 + sample; el.a14 += el.a15 + sample; el.a15 += el.a16 + sample; } for(int i = 0; i < sample_size; i++) { - moa->s4[i].a16 += moa->s5[i].a17 + sample; + maos->s4[i].a16 += maos->s5[i].a17 + sample; } - for(auto& el : moa->s5) { + for(auto& el : maos->s5) { el.a17 += el.a18 + sample; el.a18 += el.a19 + sample; el.a19 += el.a20 + sample; } - moa_many_for.sample_time(start); + maos_many_for.sample_time(start); } - samples.moa_many_for.push_back(moa_many_for); + samples.maos_many_for.push_back(maos_many_for); } - void test_soa_performance() { + void test_all_performance() { Samples samples; - int sample_size = 1000; - int sample_count = 1000; + int sample_size = 100; + int sample_count = 100; for(int i = 0; i < 100; i++) { // quick and dirty "randomness" if(i % 2 == 0) { test_large_soa(samples, sample_size, sample_count); test_large_struct(samples, sample_size, sample_count); - test_moa_style(samples, sample_count, sample_count); + test_maos_style(samples, sample_count, sample_count); } else if(i % 3 == 0) { test_large_struct(samples, sample_size, sample_count); test_large_soa(samples, sample_size, sample_count); - test_moa_style(samples, sample_count, sample_count); + test_maos_style(samples, sample_count, sample_count); } else { - test_moa_style(samples, sample_count, sample_count); + test_maos_style(samples, sample_count, sample_count); test_large_struct(samples, sample_size, sample_count); test_large_soa(samples, sample_size, sample_count); } } - fmt::println("soa_bf soa_mf ls_bf ls_mf moa_bf moa_mf"); + fmt::println("soa_bf soa_mf ls_bf ls_mf maos_bf maos_mf"); + + Samples mofm; + + Stats soa_big_for; + Stats soa_many_for; + Stats ls_big_for; + Stats ls_many_for; + Stats maos_big_for; + Stats maos_many_for; for(size_t i = 0; i < samples.soa_big_for.size(); i++) { fmt::println("{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}", @@ -416,15 +425,37 @@ namespace perf_tests { samples.soa_many_for[i].mean(), samples.ls_big_for[i].mean(), samples.ls_many_for[i].mean(), - samples.moa_big_for[i].mean(), - samples.moa_many_for[i].mean()); + samples.maos_big_for[i].mean(), + samples.maos_many_for[i].mean()); + + // this is potentially total bullshit + soa_big_for.sample(samples.soa_big_for[i].mean()); + soa_many_for.sample(samples.soa_many_for[i].mean()); + ls_big_for.sample(samples.ls_big_for[i].mean()); + ls_many_for.sample(samples.ls_many_for[i].mean()); + maos_big_for.sample(samples.maos_big_for[i].mean()); + maos_many_for.sample(samples.maos_many_for[i].mean()); } + + // totally not sure if this t-test is legit, must study more + soa_big_for.t_test(ls_big_for); + soa_big_for.t_test(maos_big_for); + + soa_many_for.t_test(ls_many_for); + soa_many_for.t_test(maos_many_for); + + ls_many_for.t_test(maos_many_for); + ls_big_for.t_test(maos_big_for); + + // the most pathologic case + fmt::println("---- {}", soa_big_for.mean() - ls_many_for.mean()); + soa_big_for.t_test(ls_many_for); } fuc2::Set TESTS{ .name="perf", .tests={ - TEST(test_soa_performance), + TEST(test_all_performance), } }; }