mamba : recurrent inference WORKS!!!

Author: compilade

ggml.c

@@ -5148,8 +5148,6 @@ static struct ggml_tensor * ggml_soft_plus_impl(
         struct ggml_tensor  * a,
         bool                  inplace) {
 
-    // TODO: does `a` need to be contiguous?
-
     bool is_node = false;
 
     if (a->grad) {
@@ -11736,7 +11734,7 @@ static void ggml_compute_forward_soft_plus_f32(
         float * x = (float *) ((char *) dst->data  + i*( dst->nb[1]));
         float * y = (float *) ((char *) src0->data + i*(src0->nb[1]));
         for (int j = 0; j < nc; ++j) {
-            x[j] = logf(1.0f + expf(y[i]));
+            x[j] = logf(1.0f + expf(y[j]));
         }
     }
 }

llama.cpp

@@ -6946,7 +6946,6 @@ struct llm_build_context {
             cur = llm_build_norm(ctx0, inpL, hparams,
                     model.layers[il].attn_norm, NULL,
                     LLM_NORM_RMS, cb, il);
-            // TODO: that's probably the wrong name.
             cb(cur, "attn_norm", il);
 
             // {n_embd, batch} * {n_embd, 2*d_inner} = {batch, 2*d_inner}