Fix: Wire remaining kernels in 5 places each

This brings the current state of the project to 2'000
kernels including both SIMD and serial endpoints.

Author: ashvardanian

CONTRIBUTING.md

@@ -357,3 +357,21 @@ To add a new operation family, for example `foo`:
 7. __Cross-platform tests__: add entries to `test/test_cross.hpp` and the relevant `test_cross_*.cpp` files.
 8. __CMakeLists.txt__: wire the new source files into the `nk_test` and `nk_bench` targets.
 9. __Language bindings__: update `python/numkong.c`, `javascript/numkong.c`, `rust/numkong.rs`, etc. as needed.
+
+## Adding a Backend Kernel to an Existing Family
+
+For primary kernels, every backend implementation should be wired in five places beyond the backend header itself:
+
+1. __Forward declaration__: add the `NK_PUBLIC` declaration with the matching `@copydoc` in the first half of `include/numkong/<family>.h`.
+2. __Compile-time dispatch__: add the `#if !NK_DYNAMIC_DISPATCH` branch in the second half of `include/numkong/<family>.h`.
+3. __Run-time dispatch__: add the dtype-specific entry to the relevant `c/dispatch_*.c` table.
+4. __Precision tests__: register the kernel in `nk_test`, usually in the existing `test/test_<family>.cpp` suite.
+5. __Benchmarks__: register the kernel in `nk_bench`, usually in the existing `bench/bench_<family>.cpp` suite.
+
+Use the existing family suite unless the kernel introduces a genuinely new test shape.
+The rule is about coverage and reachability, not about creating a brand new source file for every symbol.
+
+There are two intentional exceptions:
+
+- `cast`: the family-level `nk_cast_*` kernels follow the same header/dispatch/test/bench rule, but scalar conversion helpers are wired through `c/dispatch_other.c` and are covered through `test/test_cast.cpp` and `bench/bench_cast.cpp`.
+- `scalar`: scalar helpers are centrally declared in `include/numkong/scalar.h`, wired through `c/dispatch_other.c`, and currently do not follow the per-helper `nk_test` and `nk_bench` registration pattern.

c/dispatch_bf16.c

@@ -86,6 +86,9 @@ void nk_dispatch_bf16_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
         case nk_kernel_euclidean_k: *m = (m_t)&nk_euclidean_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
         case nk_kernel_bilinear_k: *m = (m_t)&nk_bilinear_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
         case nk_kernel_mahalanobis_k: *m = (m_t)&nk_mahalanobis_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
+        case nk_kernel_rmsd_k: *m = (m_t)&nk_rmsd_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
+        case nk_kernel_kabsch_k: *m = (m_t)&nk_kabsch_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
+        case nk_kernel_umeyama_k: *m = (m_t)&nk_umeyama_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
         case nk_kernel_each_fma_k: *m = (m_t)&nk_each_fma_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
         case nk_kernel_each_blend_k: *m = (m_t)&nk_each_blend_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
         case nk_kernel_each_scale_k: *m = (m_t)&nk_each_scale_bf16_neonbfdot, *c = nk_cap_neonbfdot_k; return;
@@ -229,6 +232,9 @@ void nk_dispatch_bf16_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
         case nk_kernel_euclidean_k: *m = (m_t)&nk_euclidean_bf16_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_bilinear_k: *m = (m_t)&nk_bilinear_bf16_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_mahalanobis_k: *m = (m_t)&nk_mahalanobis_bf16_haswell, *c = nk_cap_haswell_k; return;
+        case nk_kernel_rmsd_k: *m = (m_t)&nk_rmsd_bf16_haswell, *c = nk_cap_haswell_k; return;
+        case nk_kernel_kabsch_k: *m = (m_t)&nk_kabsch_bf16_haswell, *c = nk_cap_haswell_k; return;
+        case nk_kernel_umeyama_k: *m = (m_t)&nk_umeyama_bf16_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_each_fma_k: *m = (m_t)&nk_each_fma_bf16_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_each_blend_k: *m = (m_t)&nk_each_blend_bf16_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_each_scale_k: *m = (m_t)&nk_each_scale_bf16_haswell, *c = nk_cap_haswell_k; return;

c/dispatch_e4m3.c

@@ -53,6 +53,7 @@ void nk_dispatch_e4m3_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
 #endif
 #if NK_TARGET_NEONFHM
     if (v & nk_cap_neonfhm_k) switch (k) {
+        case nk_kernel_dot_k: *m = (m_t)&nk_dot_e4m3_neonfhm, *c = nk_cap_neonfhm_k; return;
         case nk_kernel_reduce_moments_k: *m = (m_t)&nk_reduce_moments_e4m3_neonfhm, *c = nk_cap_neonfhm_k; return;
         case nk_kernel_reduce_minmax_k: *m = (m_t)&nk_reduce_minmax_e4m3_neonfhm, *c = nk_cap_neonfhm_k; return;
         case nk_kernel_dots_packed_size_k: *m = (m_t)&nk_dots_packed_size_e4m3_neonfhm, *c = nk_cap_neonfhm_k; return;
@@ -70,6 +71,12 @@ void nk_dispatch_e4m3_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
         default: break;
         }
 #endif
+#if NK_TARGET_NEONBFDOT
+    if (v & nk_cap_neonbfdot_k) switch (k) {
+        case nk_kernel_dot_k: *m = (m_t)&nk_dot_e4m3_neonbfdot, *c = nk_cap_neonbfdot_k; return;
+        default: break;
+        }
+#endif
 #if NK_TARGET_NEON
     if (v & nk_cap_neon_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_e4m3_neon, *c = nk_cap_neon_k; return;

c/dispatch_e5m2.c

@@ -53,6 +53,7 @@ void nk_dispatch_e5m2_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
 #endif
 #if NK_TARGET_NEONFHM
     if (v & nk_cap_neonfhm_k) switch (k) {
+        case nk_kernel_dot_k: *m = (m_t)&nk_dot_e5m2_neonfhm, *c = nk_cap_neonfhm_k; return;
         case nk_kernel_reduce_moments_k: *m = (m_t)&nk_reduce_moments_e5m2_neonfhm, *c = nk_cap_neonfhm_k; return;
         case nk_kernel_reduce_minmax_k: *m = (m_t)&nk_reduce_minmax_e5m2_neonfhm, *c = nk_cap_neonfhm_k; return;
         case nk_kernel_dots_packed_size_k: *m = (m_t)&nk_dots_packed_size_e5m2_neonfhm, *c = nk_cap_neonfhm_k; return;
@@ -70,6 +71,12 @@ void nk_dispatch_e5m2_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
         default: break;
         }
 #endif
+#if NK_TARGET_NEONBFDOT
+    if (v & nk_cap_neonbfdot_k) switch (k) {
+        case nk_kernel_dot_k: *m = (m_t)&nk_dot_e5m2_neonbfdot, *c = nk_cap_neonbfdot_k; return;
+        default: break;
+        }
+#endif
 #if NK_TARGET_NEON
     if (v & nk_cap_neon_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_e5m2_neon, *c = nk_cap_neon_k; return;

c/dispatch_f32.c

@@ -48,6 +48,9 @@ void nk_dispatch_f32_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_punn
             return;
         case nk_kernel_maxsim_pack_k: *m = (m_t)&nk_maxsim_pack_f32_v128relaxed, *c = nk_cap_v128relaxed_k; return;
         case nk_kernel_maxsim_packed_k: *m = (m_t)&nk_maxsim_packed_f32_v128relaxed, *c = nk_cap_v128relaxed_k; return;
+        case nk_kernel_rmsd_k: *m = (m_t)&nk_rmsd_f32_v128relaxed, *c = nk_cap_v128relaxed_k; return;
+        case nk_kernel_kabsch_k: *m = (m_t)&nk_kabsch_f32_v128relaxed, *c = nk_cap_v128relaxed_k; return;
+        case nk_kernel_umeyama_k: *m = (m_t)&nk_umeyama_f32_v128relaxed, *c = nk_cap_v128relaxed_k; return;
         default: break;
         }
 #endif
@@ -61,6 +64,8 @@ void nk_dispatch_f32_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_punn
 #endif
 #if NK_TARGET_SMEF64
     if (v & nk_cap_smef64_k) switch (k) {
+        case nk_kernel_bilinear_k: *m = (m_t)&nk_bilinear_f32_smef64, *c = nk_cap_smef64_k; return;
+        case nk_kernel_mahalanobis_k: *m = (m_t)&nk_mahalanobis_f32_smef64, *c = nk_cap_smef64_k; return;
         case nk_kernel_dots_packed_size_k: *m = (m_t)&nk_dots_packed_size_f32_smef64, *c = nk_cap_smef64_k; return;
         case nk_kernel_dots_pack_k: *m = (m_t)&nk_dots_pack_f32_smef64, *c = nk_cap_smef64_k; return;
         case nk_kernel_dots_packed_k: *m = (m_t)&nk_dots_packed_f32_smef64, *c = nk_cap_smef64_k; return;

c/dispatch_f32c.c

@@ -15,6 +15,12 @@ void nk_dispatch_f32c_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
         default: break;
         }
 #endif
+#if NK_TARGET_SMEF64
+    if (v & nk_cap_smef64_k) switch (k) {
+        case nk_kernel_bilinear_k: *m = (m_t)&nk_bilinear_f32c_smef64, *c = nk_cap_smef64_k; return;
+        default: break;
+        }
+#endif
 #if NK_TARGET_SVE
     if (v & nk_cap_sve_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_f32c_sve, *c = nk_cap_sve_k; return;

c/dispatch_f64.c

@@ -43,11 +43,16 @@ void nk_dispatch_f64_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_punn
         case nk_kernel_euclideans_symmetric_k:
             *m = (m_t)&nk_euclideans_symmetric_f64_v128relaxed, *c = nk_cap_v128relaxed_k;
             return;
+        case nk_kernel_rmsd_k: *m = (m_t)&nk_rmsd_f64_v128relaxed, *c = nk_cap_v128relaxed_k; return;
+        case nk_kernel_kabsch_k: *m = (m_t)&nk_kabsch_f64_v128relaxed, *c = nk_cap_v128relaxed_k; return;
+        case nk_kernel_umeyama_k: *m = (m_t)&nk_umeyama_f64_v128relaxed, *c = nk_cap_v128relaxed_k; return;
         default: break;
         }
 #endif
 #if NK_TARGET_SMEF64
     if (v & nk_cap_smef64_k) switch (k) {
+        case nk_kernel_bilinear_k: *m = (m_t)&nk_bilinear_f64_smef64, *c = nk_cap_smef64_k; return;
+        case nk_kernel_mahalanobis_k: *m = (m_t)&nk_mahalanobis_f64_smef64, *c = nk_cap_smef64_k; return;
         case nk_kernel_dots_packed_size_k: *m = (m_t)&nk_dots_packed_size_f64_smef64, *c = nk_cap_smef64_k; return;
         case nk_kernel_dots_pack_k: *m = (m_t)&nk_dots_pack_f64_smef64, *c = nk_cap_smef64_k; return;
         case nk_kernel_dots_packed_k: *m = (m_t)&nk_dots_packed_f64_smef64, *c = nk_cap_smef64_k; return;

c/dispatch_f64c.c

@@ -15,6 +15,12 @@ void nk_dispatch_f64c_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
         default: break;
         }
 #endif
+#if NK_TARGET_SMEF64
+    if (v & nk_cap_smef64_k) switch (k) {
+        case nk_kernel_bilinear_k: *m = (m_t)&nk_bilinear_f64c_smef64, *c = nk_cap_smef64_k; return;
+        default: break;
+        }
+#endif
 #if NK_TARGET_SVE
     if (v & nk_cap_sve_k) switch (k) {
         case nk_kernel_dot_k: *m = (m_t)&nk_dot_f64c_sve, *c = nk_cap_sve_k; return;
@@ -55,6 +61,8 @@ void nk_dispatch_f64c_find_(nk_capability_t v, nk_kernel_kind_t k, nk_kernel_pun
 #endif
 #if NK_TARGET_HASWELL
     if (v & nk_cap_haswell_k) switch (k) {
+        case nk_kernel_dot_k: *m = (m_t)&nk_dot_f64c_haswell, *c = nk_cap_haswell_k; return;
+        case nk_kernel_vdot_k: *m = (m_t)&nk_vdot_f64c_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_each_scale_k: *m = (m_t)&nk_each_scale_f64c_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_each_blend_k: *m = (m_t)&nk_each_blend_f64c_haswell, *c = nk_cap_haswell_k; return;
         case nk_kernel_each_fma_k: *m = (m_t)&nk_each_fma_f64c_haswell, *c = nk_cap_haswell_k; return;

include/numkong/curved.h

@@ -209,6 +209,15 @@ NK_PUBLIC void nk_bilinear_f32c_smef64(nk_f32c_t const *a, nk_f32c_t const *b, n
 /** @copydoc nk_mahalanobis_f32 */
 NK_PUBLIC void nk_mahalanobis_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n,
                                          nk_f64_t *result);
+/** @copydoc nk_bilinear_f64 */
+NK_PUBLIC void nk_bilinear_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
+                                      nk_f64_t *result);
+/** @copydoc nk_bilinear_f64c */
+NK_PUBLIC void nk_bilinear_f64c_smef64(nk_f64c_t const *a, nk_f64c_t const *b, nk_f64c_t const *c, nk_size_t n,
+                                       nk_f64c_t *result);
+/** @copydoc nk_mahalanobis_f64 */
+NK_PUBLIC void nk_mahalanobis_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
+                                         nk_f64_t *result);
 #endif // NK_TARGET_SMEF64
 
 #if NK_TARGET_HASWELL
@@ -345,6 +354,8 @@ extern "C" {
 NK_PUBLIC void nk_bilinear_f64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n, nk_f64_t *result) {
 #if NK_TARGET_SKYLAKE
     nk_bilinear_f64_skylake(a, b, c, n, result);
+#elif NK_TARGET_SMEF64
+    nk_bilinear_f64_smef64(a, b, c, n, result);
 #elif NK_TARGET_RVV
     nk_bilinear_f64_rvv(a, b, c, n, result);
 #else
@@ -355,6 +366,8 @@ NK_PUBLIC void nk_bilinear_f64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t co
 NK_PUBLIC void nk_bilinear_f32(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n, nk_f64_t *result) {
 #if NK_TARGET_SKYLAKE
     nk_bilinear_f32_skylake(a, b, c, n, result);
+#elif NK_TARGET_SMEF64
+    nk_bilinear_f32_smef64(a, b, c, n, result);
 #elif NK_TARGET_HASWELL
     nk_bilinear_f32_haswell(a, b, c, n, result);
 #elif NK_TARGET_NEON
@@ -397,6 +410,8 @@ NK_PUBLIC void nk_bilinear_f64c(nk_f64c_t const *a, nk_f64c_t const *b, nk_f64c_
                                 nk_f64c_t *results) {
 #if NK_TARGET_SKYLAKE
     nk_bilinear_f64c_skylake(a, b, c, n, results);
+#elif NK_TARGET_SMEF64
+    nk_bilinear_f64c_smef64(a, b, c, n, results);
 #else
     nk_bilinear_f64c_serial(a, b, c, n, results);
 #endif
@@ -406,6 +421,8 @@ NK_PUBLIC void nk_bilinear_f32c(nk_f32c_t const *a, nk_f32c_t const *b, nk_f32c_
                                 nk_f64c_t *results) {
 #if NK_TARGET_SKYLAKE
     nk_bilinear_f32c_skylake(a, b, c, n, results);
+#elif NK_TARGET_SMEF64
+    nk_bilinear_f32c_smef64(a, b, c, n, results);
 #elif NK_TARGET_NEON
     nk_bilinear_f32c_neon(a, b, c, n, results);
 #else
@@ -437,6 +454,8 @@ NK_PUBLIC void nk_mahalanobis_f64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t
                                   nk_f64_t *result) {
 #if NK_TARGET_SKYLAKE
     nk_mahalanobis_f64_skylake(a, b, c, n, result);
+#elif NK_TARGET_SMEF64
+    nk_mahalanobis_f64_smef64(a, b, c, n, result);
 #elif NK_TARGET_RVV
     nk_mahalanobis_f64_rvv(a, b, c, n, result);
 #else
@@ -448,6 +467,8 @@ NK_PUBLIC void nk_mahalanobis_f32(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t
                                   nk_f64_t *result) {
 #if NK_TARGET_SKYLAKE
     nk_mahalanobis_f32_skylake(a, b, c, n, result);
+#elif NK_TARGET_SMEF64
+    nk_mahalanobis_f32_smef64(a, b, c, n, result);
 #elif NK_TARGET_HASWELL
     nk_mahalanobis_f32_haswell(a, b, c, n, result);
 #elif NK_TARGET_NEON

include/numkong/dot.h

@@ -307,6 +307,10 @@ NK_PUBLIC void nk_vdot_f16c_neonhalf(nk_f16c_t const *a, nk_f16c_t const *b, nk_
 #if NK_TARGET_NEONFHM
 /** @copydoc nk_dot_f16 */
 NK_PUBLIC void nk_dot_f16_neonfhm(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result);
+/** @copydoc nk_dot_e4m3 */
+NK_PUBLIC void nk_dot_e4m3_neonfhm(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result);
+/** @copydoc nk_dot_e5m2 */
+NK_PUBLIC void nk_dot_e5m2_neonfhm(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result);
 /** @copydoc nk_dot_f16c */
 NK_PUBLIC void nk_dot_f16c_neonfhm(nk_f16c_t const *a, nk_f16c_t const *b, nk_size_t n, nk_f32c_t *result);
 /** @copydoc nk_vdot_f16c */
@@ -318,6 +322,10 @@ NK_PUBLIC void nk_vdot_f16c_neonfhm(nk_f16c_t const *a, nk_f16c_t const *b, nk_s
 NK_PUBLIC void nk_dot_i8_neonsdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i32_t *result);
 /** @copydoc nk_dot_u8 */
 NK_PUBLIC void nk_dot_u8_neonsdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u32_t *result);
+/** @copydoc nk_dot_i4 */
+NK_PUBLIC void nk_dot_i4_neonsdot(nk_i4x2_t const *a, nk_i4x2_t const *b, nk_size_t n, nk_i32_t *result);
+/** @copydoc nk_dot_u4 */
+NK_PUBLIC void nk_dot_u4_neonsdot(nk_u4x2_t const *a, nk_u4x2_t const *b, nk_size_t n, nk_u32_t *result);
 /** @copydoc nk_dot_e2m3 */
 NK_PUBLIC void nk_dot_e2m3_neonsdot(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result);
 /** @copydoc nk_dot_e3m2 */
@@ -327,6 +335,10 @@ NK_PUBLIC void nk_dot_e3m2_neonsdot(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_s
 #if NK_TARGET_NEONBFDOT
 /** @copydoc nk_dot_bf16 */
 NK_PUBLIC void nk_dot_bf16_neonbfdot(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *result);
+/** @copydoc nk_dot_e4m3 */
+NK_PUBLIC void nk_dot_e4m3_neonbfdot(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result);
+/** @copydoc nk_dot_e5m2 */
+NK_PUBLIC void nk_dot_e5m2_neonbfdot(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result);
 /** @copydoc nk_dot_bf16c */
 NK_PUBLIC void nk_dot_bf16c_neonbfdot(nk_bf16c_t const *a, nk_bf16c_t const *b, nk_size_t n, nk_f32c_t *result);
 /** @copydoc nk_vdot_bf16c */
@@ -371,6 +383,10 @@ NK_PUBLIC void nk_dot_f64_haswell(nk_f64_t const *a, nk_f64_t const *b, nk_size_
 NK_PUBLIC void nk_dot_f32c_haswell(nk_f32c_t const *a, nk_f32c_t const *b, nk_size_t n, nk_f64c_t *result);
 /** @copydoc nk_vdot_f32c */
 NK_PUBLIC void nk_vdot_f32c_haswell(nk_f32c_t const *a, nk_f32c_t const *b, nk_size_t n, nk_f64c_t *result);
+/** @copydoc nk_dot_f64c */
+NK_PUBLIC void nk_dot_f64c_haswell(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n, nk_f64c_t *result);
+/** @copydoc nk_vdot_f64c */
+NK_PUBLIC void nk_vdot_f64c_haswell(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n, nk_f64c_t *result);
 
 /** @copydoc nk_dot_f16 */
 NK_PUBLIC void nk_dot_f16_haswell(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result);
@@ -694,6 +710,8 @@ NK_PUBLIC void nk_dot_u8(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u32
 NK_PUBLIC void nk_dot_i4(nk_i4x2_t const *a, nk_i4x2_t const *b, nk_size_t n, nk_i32_t *result) {
 #if NK_TARGET_ICELAKE
     nk_dot_i4_icelake(a, b, n, result);
+#elif NK_TARGET_NEONSDOT
+    nk_dot_i4_neonsdot(a, b, n, result);
 #elif NK_TARGET_RVV
     nk_dot_i4_rvv(a, b, n, result);
 #elif NK_TARGET_HASWELL
@@ -708,6 +726,8 @@ NK_PUBLIC void nk_dot_i4(nk_i4x2_t const *a, nk_i4x2_t const *b, nk_size_t n, nk
 NK_PUBLIC void nk_dot_u4(nk_u4x2_t const *a, nk_u4x2_t const *b, nk_size_t n, nk_u32_t *result) {
 #if NK_TARGET_ICELAKE
     nk_dot_u4_icelake(a, b, n, result);
+#elif NK_TARGET_NEONSDOT
+    nk_dot_u4_neonsdot(a, b, n, result);
 #elif NK_TARGET_RVV
     nk_dot_u4_rvv(a, b, n, result);
 #elif NK_TARGET_HASWELL
@@ -788,6 +808,10 @@ NK_PUBLIC void nk_dot_bf16(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n,
 NK_PUBLIC void nk_dot_e4m3(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
 #if NK_TARGET_GENOA
     nk_dot_e4m3_genoa(a, b, n, result);
+#elif NK_TARGET_NEONBFDOT
+    nk_dot_e4m3_neonbfdot(a, b, n, result);
+#elif NK_TARGET_NEONFHM
+    nk_dot_e4m3_neonfhm(a, b, n, result);
 #elif NK_TARGET_RVVHALF
     nk_dot_e4m3_rvvhalf(a, b, n, result);
 #elif NK_TARGET_RVVBF16
@@ -810,6 +834,10 @@ NK_PUBLIC void nk_dot_e4m3(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n,
 NK_PUBLIC void nk_dot_e5m2(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
 #if NK_TARGET_GENOA
     nk_dot_e5m2_genoa(a, b, n, result);
+#elif NK_TARGET_NEONBFDOT
+    nk_dot_e5m2_neonbfdot(a, b, n, result);
+#elif NK_TARGET_NEONFHM
+    nk_dot_e5m2_neonfhm(a, b, n, result);
 #elif NK_TARGET_RVVHALF
     nk_dot_e5m2_rvvhalf(a, b, n, result);
 #elif NK_TARGET_RVVBF16
@@ -962,6 +990,8 @@ NK_PUBLIC void nk_dot_f64c(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n,
     nk_dot_f64c_rvv(a, b, n, result);
 #elif NK_TARGET_SKYLAKE
     nk_dot_f64c_skylake(a, b, n, result);
+#elif NK_TARGET_HASWELL
+    nk_dot_f64c_haswell(a, b, n, result);
 #elif NK_TARGET_V128RELAXED
     nk_dot_f64c_v128relaxed(a, b, n, result);
 #else
@@ -1022,6 +1052,8 @@ NK_PUBLIC void nk_vdot_f64c(nk_f64c_t const *a, nk_f64c_t const *b, nk_size_t n,
     nk_vdot_f64c_rvv(a, b, n, result);
 #elif NK_TARGET_SKYLAKE
     nk_vdot_f64c_skylake(a, b, n, result);
+#elif NK_TARGET_HASWELL
+    nk_vdot_f64c_haswell(a, b, n, result);
 #elif NK_TARGET_V128RELAXED
     nk_vdot_f64c_v128relaxed(a, b, n, result);
 #else