Decouple activations from linear layer.

5 files changed, 169 insertions, 125 deletions

diff --git a/src/lib/test/neuralnet_test.c b/src/lib/test/neuralnet_test.c
index 14d9438..0f8d7b8 100644
--- a/src/lib/test/neuralnet_test.c
+++ b/src/lib/test/neuralnet_test.c
@@ -1,8 +1,8 @@
 #include <neuralnet/neuralnet.h>
 
-#include <neuralnet/matrix.h>
 #include "activation.h"
 #include "neuralnet_impl.h"
+#include <neuralnet/matrix.h>
 
 #include "test.h"
 #include "test_util.h"
@@ -10,23 +10,31 @@
 #include <assert.h>
 
 TEST_CASE(neuralnet_perceptron_test) {
-  const int num_layers = 1;
-  const int layer_sizes[] = { 1, 1 };
-  const nnActivation layer_activations[] = { nnSigmoid };
-  const R weights[] = { 0.3 };
+  const int     num_layers = 2;
+  const int     input_size = 1;
+  const R       weights[]  = {0.3};
+  const R       biases[]   = {0.0};
+  const nnLayer layers[]   = {
+      {.type = nnLinear,
+       .linear =
+             {.weights = nnMatrixFromArray(1, 1, weights),
+              .biases  = nnMatrixFromArray(1, 1, biases)}},
+      {.type = nnSigmoid},
+  };
 
-  nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+  nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
   assert(net);
-  nnSetWeights(net, weights);
 
-  nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+  nnQueryObject* query = nnMakeQueryObject(net, 1);
 
-  const R input[] = { 0.9 };
-  R output[1];
+  const R input[] = {0.9};
+  R       output[1];
   nnQueryArray(net, query, input, output);
 
   const R expected_output = sigmoid(input[0] * weights[0]);
-  printf("\nOutput: %f, Expected: %f\n", output[0], expected_output);
+  printf(
+      "\n[neuralnet_perceptron_test] Output: %f, Expected: %f\n", output[0],
+      expected_output);
   TEST_TRUE(double_eq(output[0], expected_output, EPS));
 
   nnDeleteQueryObject(&query);
@@ -34,53 +42,66 @@ TEST_CASE(neuralnet_perceptron_test) {
 }
 
 TEST_CASE(neuralnet_xor_test) {
-  const int num_layers = 2;
-  const int layer_sizes[] = { 2, 2, 1 };
-  const nnActivation layer_activations[] = { nnRelu, nnIdentity };
-  const R weights[] = {
-    1, 1, 1, 1,  // First (hidden) layer.
-    1, -2        // Second (output) layer.
-  };
-  const R biases[] = {
-    0, -1,  // First (hidden) layer.
-    0       // Second (output) layer.
+  // First (hidden) layer.
+  const R weights0[] = {1, 1, 1, 1};
+  const R biases0[]  = {0, -1};
+  // Second (output) layer.
+  const R weights1[] = {1, -2};
+  const R biases1[]  = {0};
+  // Network.
+  const int     num_layers = 3;
+  const int     input_size = 2;
+  const nnLayer layers[]   = {
+      {.type = nnLinear,
+       .linear =
+             {.weights = nnMatrixFromArray(2, 2, weights0),
+              .biases  = nnMatrixFromArray(1, 2, biases0)}},
+      {.type = nnRelu},
+      {.type = nnLinear,
+       .linear =
+             {.weights = nnMatrixFromArray(2, 1, weights1),
+              .biases  = nnMatrixFromArray(1, 1, biases1)}},
   };
 
-  nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+  nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
   assert(net);
-  nnSetWeights(net, weights);
-  nnSetBiases(net, biases);
 
   // First layer weights.
-  TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 0), 1);
-  TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 1), 1);
-  TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 2), 1);
-  TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 3), 1);
-  // Second layer weights.
-  TEST_EQUAL(nnMatrixAt(&net->weights[1], 0, 0), 1);
-  TEST_EQUAL(nnMatrixAt(&net->weights[1], 0, 1), -2);
+  TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 0), 1);
+  TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 1), 1);
+  TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 2), 1);
+  TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 3), 1);
+  // Second linear layer (third layer) weights.
+  TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.weights, 0, 0), 1);
+  TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.weights, 0, 1), -2);
   // First layer biases.
-  TEST_EQUAL(nnMatrixAt(&net->biases[0], 0, 0), 0);
-  TEST_EQUAL(nnMatrixAt(&net->biases[0], 0, 1), -1);
-  // Second layer biases.
-  TEST_EQUAL(nnMatrixAt(&net->biases[1], 0, 0), 0);
+  TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.biases, 0, 0), 0);
+  TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.biases, 0, 1), -1);
+  // Second linear layer (third layer) biases.
+  TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.biases, 0, 0), 0);
 
   // Test.
 
-  #define M 4
+#define M 4
 
-  nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/M);
+  nnQueryObject* query = nnMakeQueryObject(net, M);
 
-  const R test_inputs[M][2] = { { 0., 0. }, { 1., 0. }, { 0., 1. }, { 1., 1. } };
+  const R test_inputs[M][2] = {
+      {0., 0.},
+      {1., 0.},
+      {0., 1.},
+      {1., 1.}
+  };
   nnMatrix test_inputs_matrix = nnMatrixMake(M, 2);
   nnMatrixInit(&test_inputs_matrix, (const R*)test_inputs);
   nnQuery(net, query, &test_inputs_matrix);
 
-  const R expected_outputs[M] = { 0., 1., 1., 0. };
+  const R expected_outputs[M] = {0., 1., 1., 0.};
   for (int i = 0; i < M; ++i) {
     const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
-    printf("\nInput: (%f, %f), Output: %f, Expected: %f\n",
-      test_inputs[i][0], test_inputs[i][1], test_output, expected_outputs[i]);
+    printf(
+        "\nInput: (%f, %f), Output: %f, Expected: %f\n", test_inputs[i][0],
+        test_inputs[i][1], test_output, expected_outputs[i]);
   }
   for (int i = 0; i < M; ++i) {
     const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
diff --git a/src/lib/test/train_linear_perceptron_non_origin_test.c b/src/lib/test/train_linear_perceptron_non_origin_test.c
index 5a320ac..40a42e0 100644
--- a/src/lib/test/train_linear_perceptron_non_origin_test.c
+++ b/src/lib/test/train_linear_perceptron_non_origin_test.c
@@ -1,9 +1,8 @@
 #include <neuralnet/train.h>
 
+#include "neuralnet_impl.h"
 #include <neuralnet/matrix.h>
 #include <neuralnet/neuralnet.h>
-#include "activation.h"
-#include "neuralnet_impl.h"
 
 #include "test.h"
 #include "test_util.h"
@@ -11,19 +10,21 @@
 #include <assert.h>
 
 TEST_CASE(neuralnet_train_linear_perceptron_non_origin_test) {
-  const int num_layers = 1;
-  const int layer_sizes[] = { 1, 1 };
-  const nnActivation layer_activations[] = { nnIdentity };
+  const int     num_layers = 1;
+  const int     input_size = 1;
+  const nnLayer layers[]   = {
+      {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}}
+  };
 
-  nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+  nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
   assert(net);
 
-  // Train.
+// Train.
 
-  // Try to learn the Y = 2X + 1 line.
-  #define N 2
-  const R inputs[N]  = { 0., 1. };
-  const R targets[N] = { 1., 3. };
+// Try to learn the Y = 2X + 1 line.
+#define N 2
+  const R inputs[N]  = {0., 1.};
+  const R targets[N] = {1., 3.};
 
   nnMatrix inputs_matrix  = nnMatrixMake(N, 1);
   nnMatrix targets_matrix = nnMatrixMake(N, 1);
@@ -31,31 +32,32 @@ TEST_CASE(neuralnet_train_linear_perceptron_non_origin_test) {
   nnMatrixInit(&targets_matrix, targets);
 
   nnTrainingParams params = {
-    .learning_rate = 0.7,
-    .max_iterations = 20,
-    .seed = 0,
-    .weight_init = nnWeightInit01,
-    .debug = false,
+      .learning_rate  = 0.7,
+      .max_iterations = 20,
+      .seed           = 0,
+      .weight_init    = nnWeightInit01,
+      .debug          = false,
   };
 
   nnTrain(net, &inputs_matrix, &targets_matrix, &params);
 
-  const R weight = nnMatrixAt(&net->weights[0], 0, 0);
+  const R weight          = nnMatrixAt(&net->layers[0].linear.weights, 0, 0);
   const R expected_weight = 2.0;
-  printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
+  printf(
+      "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
   TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS));
 
-  const R bias = nnMatrixAt(&net->biases[0], 0, 0);
+  const R bias          = nnMatrixAt(&net->layers[0].linear.biases, 0, 0);
   const R expected_bias = 1.0;
   printf("Trained network bias: %f, Expected: %f\n", bias, expected_bias);
   TEST_TRUE(double_eq(bias, expected_bias, WEIGHT_EPS));
 
   // Test.
 
-  nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+  nnQueryObject* query = nnMakeQueryObject(net, 1);
 
-  const R test_input[] = { 2.3 };
-  R test_output[1];
+  const R test_input[] = {2.3};
+  R       test_output[1];
   nnQueryArray(net, query, test_input, test_output);
 
   const R expected_output = test_input[0] * expected_weight + expected_bias;
diff --git a/src/lib/test/train_linear_perceptron_test.c b/src/lib/test/train_linear_perceptron_test.c
index 2b1336d..667643b 100644
--- a/src/lib/test/train_linear_perceptron_test.c
+++ b/src/lib/test/train_linear_perceptron_test.c
@@ -1,9 +1,8 @@
 #include <neuralnet/train.h>
 
+#include "neuralnet_impl.h"
 #include <neuralnet/matrix.h>
 #include <neuralnet/neuralnet.h>
-#include "activation.h"
-#include "neuralnet_impl.h"
 
 #include "test.h"
 #include "test_util.h"
@@ -11,19 +10,21 @@
 #include <assert.h>
 
 TEST_CASE(neuralnet_train_linear_perceptron_test) {
-  const int num_layers = 1;
-  const int layer_sizes[] = { 1, 1 };
-  const nnActivation layer_activations[] = { nnIdentity };
+  const int     num_layers = 1;
+  const int     input_size = 1;
+  const nnLayer layers[]   = {
+      {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}}
+  };
 
-  nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+  nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
   assert(net);
 
-  // Train.
+// Train.
 
-  // Try to learn the Y=X line.
-  #define N 2
-  const R inputs[N]  = { 0., 1. };
-  const R targets[N] = { 0., 1. };
+// Try to learn the Y=X line.
+#define N 2
+  const R inputs[N]  = {0., 1.};
+  const R targets[N] = {0., 1.};
 
   nnMatrix inputs_matrix  = nnMatrixMake(N, 1);
   nnMatrix targets_matrix = nnMatrixMake(N, 1);
@@ -31,26 +32,27 @@ TEST_CASE(neuralnet_train_linear_perceptron_test) {
   nnMatrixInit(&targets_matrix, targets);
 
   nnTrainingParams params = {
-    .learning_rate = 0.7,
-    .max_iterations = 10,
-    .seed = 0,
-    .weight_init = nnWeightInit01,
-    .debug = false,
+      .learning_rate  = 0.7,
+      .max_iterations = 10,
+      .seed           = 0,
+      .weight_init    = nnWeightInit01,
+      .debug          = false,
   };
 
   nnTrain(net, &inputs_matrix, &targets_matrix, &params);
 
-  const R weight = nnMatrixAt(&net->weights[0], 0, 0);
+  const R weight          = nnMatrixAt(&net->layers[0].linear.weights, 0, 0);
   const R expected_weight = 1.0;
-  printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
+  printf(
+      "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
   TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS));
 
   // Test.
 
-  nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+  nnQueryObject* query = nnMakeQueryObject(net, 1);
 
-  const R test_input[] = { 2.3 };
-  R test_output[1];
+  const R test_input[] = {2.3};
+  R       test_output[1];
   nnQueryArray(net, query, test_input, test_output);
 
   const R expected_output = test_input[0];
diff --git a/src/lib/test/train_sigmoid_test.c b/src/lib/test/train_sigmoid_test.c
index 588e7ca..39a84b0 100644
--- a/src/lib/test/train_sigmoid_test.c
+++ b/src/lib/test/train_sigmoid_test.c
@@ -1,9 +1,9 @@
 #include <neuralnet/train.h>
 
-#include <neuralnet/matrix.h>
-#include <neuralnet/neuralnet.h>
 #include "activation.h"
 #include "neuralnet_impl.h"
+#include <neuralnet/matrix.h>
+#include <neuralnet/neuralnet.h>
 
 #include "test.h"
 #include "test_util.h"
@@ -11,21 +11,24 @@
 #include <assert.h>
 
 TEST_CASE(neuralnet_train_sigmoid_test) {
-  const int num_layers = 1;
-  const int layer_sizes[] = { 1, 1 };
-  const nnActivation layer_activations[] = { nnSigmoid };
+  const int     num_layers = 2;
+  const int     input_size = 1;
+  const nnLayer layers[]   = {
+      {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}},
+      {.type = nnSigmoid},
+  };
 
-  nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+  nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
   assert(net);
 
-  // Train.
+// Train.
 
-  // Try to learn the sigmoid function.
-  #define N 3
+// Try to learn the sigmoid function.
+#define N 3
   R inputs[N];
   R targets[N];
   for (int i = 0; i < N; ++i) {
-    inputs[i] = lerp(-1, +1, (R)i / (R)(N-1));
+    inputs[i]  = lerp(-1, +1, (R)i / (R)(N - 1));
     targets[i] = sigmoid(inputs[i]);
   }
 
@@ -35,29 +38,30 @@ TEST_CASE(neuralnet_train_sigmoid_test) {
   nnMatrixInit(&targets_matrix, targets);
 
   nnTrainingParams params = {
-    .learning_rate = 0.9,
-    .max_iterations = 100,
-    .seed = 0,
-    .weight_init = nnWeightInit01,
-    .debug = false,
+      .learning_rate  = 0.9,
+      .max_iterations = 100,
+      .seed           = 0,
+      .weight_init    = nnWeightInit01,
+      .debug          = false,
   };
 
   nnTrain(net, &inputs_matrix, &targets_matrix, &params);
 
-  const R weight = nnMatrixAt(&net->weights[0], 0, 0);
+  const R weight          = nnMatrixAt(&net->layers[0].linear.weights, 0, 0);
   const R expected_weight = 1.0;
-  printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
+  printf(
+      "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
   TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS));
 
   // Test.
 
-  nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+  nnQueryObject* query = nnMakeQueryObject(net, 1);
 
-  const R test_input[] = { 0.3 };
-  R test_output[1];
+  const R test_input[] = {0.3};
+  R       test_output[1];
   nnQueryArray(net, query, test_input, test_output);
 
-  const R expected_output = 0.574442516811659;  // sigmoid(0.3)
+  const R expected_output = 0.574442516811659; // sigmoid(0.3)
   printf("Output: %f, Expected: %f\n", test_output[0], expected_output);
   TEST_TRUE(double_eq(test_output[0], expected_output, OUTPUT_EPS));
 
diff --git a/src/lib/test/train_xor_test.c b/src/lib/test/train_xor_test.c
index 6ddc6e0..78695a3 100644
--- a/src/lib/test/train_xor_test.c
+++ b/src/lib/test/train_xor_test.c
@@ -1,9 +1,9 @@
 #include <neuralnet/train.h>
 
-#include <neuralnet/matrix.h>
-#include <neuralnet/neuralnet.h>
 #include "activation.h"
 #include "neuralnet_impl.h"
+#include <neuralnet/matrix.h>
+#include <neuralnet/neuralnet.h>
 
 #include "test.h"
 #include "test_util.h"
@@ -11,18 +11,27 @@
 #include <assert.h>
 
 TEST_CASE(neuralnet_train_xor_test) {
-  const int num_layers = 2;
-  const int layer_sizes[] = { 2, 2, 1 };
-  const nnActivation layer_activations[] = { nnRelu, nnIdentity };
+  const int     num_layers = 3;
+  const int     input_size = 2;
+  const nnLayer layers[]   = {
+      {.type = nnLinear, .linear = {.input_size = 2, .output_size = 2}},
+      {.type = nnRelu},
+      {.type = nnLinear, .linear = {.input_size = 2, .output_size = 1}}
+  };
 
-  nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+  nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
   assert(net);
 
   // Train.
 
-  #define N 4
-  const R inputs[N][2]  = { { 0., 0. }, { 0., 1. }, { 1., 0. }, { 1., 1. } };
-  const R targets[N] = { 0., 1., 1., 0. };
+#define N 4
+  const R inputs[N][2] = {
+      {0., 0.},
+      {0., 1.},
+      {1., 0.},
+      {1., 1.}
+  };
+  const R targets[N] = {0., 1., 1., 0.};
 
   nnMatrix inputs_matrix  = nnMatrixMake(N, 2);
   nnMatrix targets_matrix = nnMatrixMake(N, 1);
@@ -30,31 +39,37 @@ TEST_CASE(neuralnet_train_xor_test) {
   nnMatrixInit(&targets_matrix, targets);
 
   nnTrainingParams params = {
-    .learning_rate = 0.1,
-    .max_iterations = 500,
-    .seed = 0,
-    .weight_init = nnWeightInit01,
-    .debug = false,
+      .learning_rate  = 0.1,
+      .max_iterations = 500,
+      .seed           = 0,
+      .weight_init    = nnWeightInit01,
+      .debug          = false,
   };
 
   nnTrain(net, &inputs_matrix, &targets_matrix, &params);
 
   // Test.
 
-  #define M 4
+#define M 4
 
-  nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/M);
+  nnQueryObject* query = nnMakeQueryObject(net, M);
 
-  const R test_inputs[M][2] = { { 0., 0. }, { 1., 0. }, { 0., 1. }, { 1., 1. } };
+  const R test_inputs[M][2] = {
+      {0., 0.},
+      {1., 0.},
+      {0., 1.},
+      {1., 1.}
+  };
   nnMatrix test_inputs_matrix = nnMatrixMake(M, 2);
   nnMatrixInit(&test_inputs_matrix, (const R*)test_inputs);
   nnQuery(net, query, &test_inputs_matrix);
 
-  const R expected_outputs[M] = { 0., 1., 1., 0. };
+  const R expected_outputs[M] = {0., 1., 1., 0.};
   for (int i = 0; i < M; ++i) {
     const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
-    printf("\nInput: (%f, %f), Output: %f, Expected: %f\n",
-      test_inputs[i][0], test_inputs[i][1], test_output, expected_outputs[i]);
+    printf(
+        "\nInput: (%f, %f), Output: %f, Expected: %f\n", test_inputs[i][0],
+        test_inputs[i][1], test_output, expected_outputs[i]);
   }
   for (int i = 0; i < M; ++i) {
     const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);