From 653e98e029a0d0f110b0ac599e50406060bb0f87 Mon Sep 17 00:00:00 2001
From: 3gg <3gg@shellblade.net>
Date: Sat, 16 Dec 2023 10:21:16 -0800
Subject: Decouple activations from linear layer.
---
src/lib/test/neuralnet_test.c | 103 +++++++++++++--------
.../test/train_linear_perceptron_non_origin_test.c | 46 ++++-----
src/lib/test/train_linear_perceptron_test.c | 44 ++++-----
src/lib/test/train_sigmoid_test.c | 46 ++++-----
src/lib/test/train_xor_test.c | 55 +++++++----
5 files changed, 169 insertions(+), 125 deletions(-)
(limited to 'src/lib/test')
diff --git a/src/lib/test/neuralnet_test.c b/src/lib/test/neuralnet_test.c
index 14d9438..0f8d7b8 100644
--- a/src/lib/test/neuralnet_test.c
+++ b/src/lib/test/neuralnet_test.c
@@ -1,8 +1,8 @@
#include <neuralnet/neuralnet.h>
-#include <neuralnet/matrix.h>
#include "activation.h"
#include "neuralnet_impl.h"
+#include <neuralnet/matrix.h>
#include "test.h"
#include "test_util.h"
@@ -10,23 +10,31 @@
#include <assert.h>
TEST_CASE(neuralnet_perceptron_test) {
- const int num_layers = 1;
- const int layer_sizes[] = { 1, 1 };
- const nnActivation layer_activations[] = { nnSigmoid };
- const R weights[] = { 0.3 };
+ const int num_layers = 2;
+ const int input_size = 1;
+ const R weights[] = {0.3};
+ const R biases[] = {0.0};
+ const nnLayer layers[] = {
+ {.type = nnLinear,
+ .linear =
+ {.weights = nnMatrixFromArray(1, 1, weights),
+ .biases = nnMatrixFromArray(1, 1, biases)}},
+ {.type = nnSigmoid},
+ };
- nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+ nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
assert(net);
- nnSetWeights(net, weights);
- nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+ nnQueryObject* query = nnMakeQueryObject(net, 1);
- const R input[] = { 0.9 };
- R output[1];
+ const R input[] = {0.9};
+ R output[1];
nnQueryArray(net, query, input, output);
const R expected_output = sigmoid(input[0] * weights[0]);
- printf("\nOutput: %f, Expected: %f\n", output[0], expected_output);
+ printf(
+ "\n[neuralnet_perceptron_test] Output: %f, Expected: %f\n", output[0],
+ expected_output);
TEST_TRUE(double_eq(output[0], expected_output, EPS));
nnDeleteQueryObject(&query);
@@ -34,53 +42,66 @@ TEST_CASE(neuralnet_perceptron_test) {
}
TEST_CASE(neuralnet_xor_test) {
- const int num_layers = 2;
- const int layer_sizes[] = { 2, 2, 1 };
- const nnActivation layer_activations[] = { nnRelu, nnIdentity };
- const R weights[] = {
- 1, 1, 1, 1, // First (hidden) layer.
- 1, -2 // Second (output) layer.
- };
- const R biases[] = {
- 0, -1, // First (hidden) layer.
- 0 // Second (output) layer.
+ // First (hidden) layer.
+ const R weights0[] = {1, 1, 1, 1};
+ const R biases0[] = {0, -1};
+ // Second (output) layer.
+ const R weights1[] = {1, -2};
+ const R biases1[] = {0};
+ // Network.
+ const int num_layers = 3;
+ const int input_size = 2;
+ const nnLayer layers[] = {
+ {.type = nnLinear,
+ .linear =
+ {.weights = nnMatrixFromArray(2, 2, weights0),
+ .biases = nnMatrixFromArray(1, 2, biases0)}},
+ {.type = nnRelu},
+ {.type = nnLinear,
+ .linear =
+ {.weights = nnMatrixFromArray(2, 1, weights1),
+ .biases = nnMatrixFromArray(1, 1, biases1)}},
};
- nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+ nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
assert(net);
- nnSetWeights(net, weights);
- nnSetBiases(net, biases);
// First layer weights.
- TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 0), 1);
- TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 1), 1);
- TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 2), 1);
- TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 3), 1);
- // Second layer weights.
- TEST_EQUAL(nnMatrixAt(&net->weights[1], 0, 0), 1);
- TEST_EQUAL(nnMatrixAt(&net->weights[1], 0, 1), -2);
+ TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 0), 1);
+ TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 1), 1);
+ TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 2), 1);
+ TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 3), 1);
+ // Second linear layer (third layer) weights.
+ TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.weights, 0, 0), 1);
+ TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.weights, 0, 1), -2);
// First layer biases.
- TEST_EQUAL(nnMatrixAt(&net->biases[0], 0, 0), 0);
- TEST_EQUAL(nnMatrixAt(&net->biases[0], 0, 1), -1);
- // Second layer biases.
- TEST_EQUAL(nnMatrixAt(&net->biases[1], 0, 0), 0);
+ TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.biases, 0, 0), 0);
+ TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.biases, 0, 1), -1);
+ // Second linear layer (third layer) biases.
+ TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.biases, 0, 0), 0);
// Test.
- #define M 4
+#define M 4
- nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/M);
+ nnQueryObject* query = nnMakeQueryObject(net, M);
- const R test_inputs[M][2] = { { 0., 0. }, { 1., 0. }, { 0., 1. }, { 1., 1. } };
+ const R test_inputs[M][2] = {
+ {0., 0.},
+ {1., 0.},
+ {0., 1.},
+ {1., 1.}
+ };
nnMatrix test_inputs_matrix = nnMatrixMake(M, 2);
nnMatrixInit(&test_inputs_matrix, (const R*)test_inputs);
nnQuery(net, query, &test_inputs_matrix);
- const R expected_outputs[M] = { 0., 1., 1., 0. };
+ const R expected_outputs[M] = {0., 1., 1., 0.};
for (int i = 0; i < M; ++i) {
const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
- printf("\nInput: (%f, %f), Output: %f, Expected: %f\n",
- test_inputs[i][0], test_inputs[i][1], test_output, expected_outputs[i]);
+ printf(
+ "\nInput: (%f, %f), Output: %f, Expected: %f\n", test_inputs[i][0],
+ test_inputs[i][1], test_output, expected_outputs[i]);
}
for (int i = 0; i < M; ++i) {
const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
diff --git a/src/lib/test/train_linear_perceptron_non_origin_test.c b/src/lib/test/train_linear_perceptron_non_origin_test.c
index 5a320ac..40a42e0 100644
--- a/src/lib/test/train_linear_perceptron_non_origin_test.c
+++ b/src/lib/test/train_linear_perceptron_non_origin_test.c
@@ -1,9 +1,8 @@
#include <neuralnet/train.h>
+#include "neuralnet_impl.h"
#include <neuralnet/matrix.h>
#include <neuralnet/neuralnet.h>
-#include "activation.h"
-#include "neuralnet_impl.h"
#include "test.h"
#include "test_util.h"
@@ -11,19 +10,21 @@
#include <assert.h>
TEST_CASE(neuralnet_train_linear_perceptron_non_origin_test) {
- const int num_layers = 1;
- const int layer_sizes[] = { 1, 1 };
- const nnActivation layer_activations[] = { nnIdentity };
+ const int num_layers = 1;
+ const int input_size = 1;
+ const nnLayer layers[] = {
+ {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}}
+ };
- nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+ nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
assert(net);
- // Train.
+// Train.
- // Try to learn the Y = 2X + 1 line.
- #define N 2
- const R inputs[N] = { 0., 1. };
- const R targets[N] = { 1., 3. };
+// Try to learn the Y = 2X + 1 line.
+#define N 2
+ const R inputs[N] = {0., 1.};
+ const R targets[N] = {1., 3.};
nnMatrix inputs_matrix = nnMatrixMake(N, 1);
nnMatrix targets_matrix = nnMatrixMake(N, 1);
@@ -31,31 +32,32 @@ TEST_CASE(neuralnet_train_linear_perceptron_non_origin_test) {
nnMatrixInit(&targets_matrix, targets);
nnTrainingParams params = {
- .learning_rate = 0.7,
- .max_iterations = 20,
- .seed = 0,
- .weight_init = nnWeightInit01,
- .debug = false,
+ .learning_rate = 0.7,
+ .max_iterations = 20,
+ .seed = 0,
+ .weight_init = nnWeightInit01,
+ .debug = false,
};
nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms);
- const R weight = nnMatrixAt(&net->weights[0], 0, 0);
+ const R weight = nnMatrixAt(&net->layers[0].linear.weights, 0, 0);
const R expected_weight = 2.0;
- printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
+ printf(
+ "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS));
- const R bias = nnMatrixAt(&net->biases[0], 0, 0);
+ const R bias = nnMatrixAt(&net->layers[0].linear.biases, 0, 0);
const R expected_bias = 1.0;
printf("Trained network bias: %f, Expected: %f\n", bias, expected_bias);
TEST_TRUE(double_eq(bias, expected_bias, WEIGHT_EPS));
// Test.
- nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+ nnQueryObject* query = nnMakeQueryObject(net, 1);
- const R test_input[] = { 2.3 };
- R test_output[1];
+ const R test_input[] = {2.3};
+ R test_output[1];
nnQueryArray(net, query, test_input, test_output);
const R expected_output = test_input[0] * expected_weight + expected_bias;
diff --git a/src/lib/test/train_linear_perceptron_test.c b/src/lib/test/train_linear_perceptron_test.c
index 2b1336d..667643b 100644
--- a/src/lib/test/train_linear_perceptron_test.c
+++ b/src/lib/test/train_linear_perceptron_test.c
@@ -1,9 +1,8 @@
#include <neuralnet/train.h>
+#include "neuralnet_impl.h"
#include <neuralnet/matrix.h>
#include <neuralnet/neuralnet.h>
-#include "activation.h"
-#include "neuralnet_impl.h"
#include "test.h"
#include "test_util.h"
@@ -11,19 +10,21 @@
#include <assert.h>
TEST_CASE(neuralnet_train_linear_perceptron_test) {
- const int num_layers = 1;
- const int layer_sizes[] = { 1, 1 };
- const nnActivation layer_activations[] = { nnIdentity };
+ const int num_layers = 1;
+ const int input_size = 1;
+ const nnLayer layers[] = {
+ {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}}
+ };
- nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+ nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
assert(net);
- // Train.
+// Train.
- // Try to learn the Y=X line.
- #define N 2
- const R inputs[N] = { 0., 1. };
- const R targets[N] = { 0., 1. };
+// Try to learn the Y=X line.
+#define N 2
+ const R inputs[N] = {0., 1.};
+ const R targets[N] = {0., 1.};
nnMatrix inputs_matrix = nnMatrixMake(N, 1);
nnMatrix targets_matrix = nnMatrixMake(N, 1);
@@ -31,26 +32,27 @@ TEST_CASE(neuralnet_train_linear_perceptron_test) {
nnMatrixInit(&targets_matrix, targets);
nnTrainingParams params = {
- .learning_rate = 0.7,
- .max_iterations = 10,
- .seed = 0,
- .weight_init = nnWeightInit01,
- .debug = false,
+ .learning_rate = 0.7,
+ .max_iterations = 10,
+ .seed = 0,
+ .weight_init = nnWeightInit01,
+ .debug = false,
};
nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms);
- const R weight = nnMatrixAt(&net->weights[0], 0, 0);
+ const R weight = nnMatrixAt(&net->layers[0].linear.weights, 0, 0);
const R expected_weight = 1.0;
- printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
+ printf(
+ "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS));
// Test.
- nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+ nnQueryObject* query = nnMakeQueryObject(net, 1);
- const R test_input[] = { 2.3 };
- R test_output[1];
+ const R test_input[] = {2.3};
+ R test_output[1];
nnQueryArray(net, query, test_input, test_output);
const R expected_output = test_input[0];
diff --git a/src/lib/test/train_sigmoid_test.c b/src/lib/test/train_sigmoid_test.c
index 588e7ca..39a84b0 100644
--- a/src/lib/test/train_sigmoid_test.c
+++ b/src/lib/test/train_sigmoid_test.c
@@ -1,9 +1,9 @@
#include <neuralnet/train.h>
-#include <neuralnet/matrix.h>
-#include <neuralnet/neuralnet.h>
#include "activation.h"
#include "neuralnet_impl.h"
+#include <neuralnet/matrix.h>
+#include <neuralnet/neuralnet.h>
#include "test.h"
#include "test_util.h"
@@ -11,21 +11,24 @@
#include <assert.h>
TEST_CASE(neuralnet_train_sigmoid_test) {
- const int num_layers = 1;
- const int layer_sizes[] = { 1, 1 };
- const nnActivation layer_activations[] = { nnSigmoid };
+ const int num_layers = 2;
+ const int input_size = 1;
+ const nnLayer layers[] = {
+ {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}},
+ {.type = nnSigmoid},
+ };
- nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+ nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
assert(net);
- // Train.
+// Train.
- // Try to learn the sigmoid function.
- #define N 3
+// Try to learn the sigmoid function.
+#define N 3
R inputs[N];
R targets[N];
for (int i = 0; i < N; ++i) {
- inputs[i] = lerp(-1, +1, (R)i / (R)(N-1));
+ inputs[i] = lerp(-1, +1, (R)i / (R)(N - 1));
targets[i] = sigmoid(inputs[i]);
}
@@ -35,29 +38,30 @@ TEST_CASE(neuralnet_train_sigmoid_test) {
nnMatrixInit(&targets_matrix, targets);
nnTrainingParams params = {
- .learning_rate = 0.9,
- .max_iterations = 100,
- .seed = 0,
- .weight_init = nnWeightInit01,
- .debug = false,
+ .learning_rate = 0.9,
+ .max_iterations = 100,
+ .seed = 0,
+ .weight_init = nnWeightInit01,
+ .debug = false,
};
nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms);
- const R weight = nnMatrixAt(&net->weights[0], 0, 0);
+ const R weight = nnMatrixAt(&net->layers[0].linear.weights, 0, 0);
const R expected_weight = 1.0;
- printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
+ printf(
+ "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight);
TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS));
// Test.
- nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1);
+ nnQueryObject* query = nnMakeQueryObject(net, 1);
- const R test_input[] = { 0.3 };
- R test_output[1];
+ const R test_input[] = {0.3};
+ R test_output[1];
nnQueryArray(net, query, test_input, test_output);
- const R expected_output = 0.574442516811659; // sigmoid(0.3)
+ const R expected_output = 0.574442516811659; // sigmoid(0.3)
printf("Output: %f, Expected: %f\n", test_output[0], expected_output);
TEST_TRUE(double_eq(test_output[0], expected_output, OUTPUT_EPS));
diff --git a/src/lib/test/train_xor_test.c b/src/lib/test/train_xor_test.c
index 6ddc6e0..78695a3 100644
--- a/src/lib/test/train_xor_test.c
+++ b/src/lib/test/train_xor_test.c
@@ -1,9 +1,9 @@
#include <neuralnet/train.h>
-#include <neuralnet/matrix.h>
-#include <neuralnet/neuralnet.h>
#include "activation.h"
#include "neuralnet_impl.h"
+#include <neuralnet/matrix.h>
+#include <neuralnet/neuralnet.h>
#include "test.h"
#include "test_util.h"
@@ -11,18 +11,27 @@
#include <assert.h>
TEST_CASE(neuralnet_train_xor_test) {
- const int num_layers = 2;
- const int layer_sizes[] = { 2, 2, 1 };
- const nnActivation layer_activations[] = { nnRelu, nnIdentity };
+ const int num_layers = 3;
+ const int input_size = 2;
+ const nnLayer layers[] = {
+ {.type = nnLinear, .linear = {.input_size = 2, .output_size = 2}},
+ {.type = nnRelu},
+ {.type = nnLinear, .linear = {.input_size = 2, .output_size = 1}}
+ };
- nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations);
+ nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size);
assert(net);
// Train.
- #define N 4
- const R inputs[N][2] = { { 0., 0. }, { 0., 1. }, { 1., 0. }, { 1., 1. } };
- const R targets[N] = { 0., 1., 1., 0. };
+#define N 4
+ const R inputs[N][2] = {
+ {0., 0.},
+ {0., 1.},
+ {1., 0.},
+ {1., 1.}
+ };
+ const R targets[N] = {0., 1., 1., 0.};
nnMatrix inputs_matrix = nnMatrixMake(N, 2);
nnMatrix targets_matrix = nnMatrixMake(N, 1);
@@ -30,31 +39,37 @@ TEST_CASE(neuralnet_train_xor_test) {
nnMatrixInit(&targets_matrix, targets);
nnTrainingParams params = {
- .learning_rate = 0.1,
- .max_iterations = 500,
- .seed = 0,
- .weight_init = nnWeightInit01,
- .debug = false,
+ .learning_rate = 0.1,
+ .max_iterations = 500,
+ .seed = 0,
+ .weight_init = nnWeightInit01,
+ .debug = false,
};
nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms);
// Test.
- #define M 4
+#define M 4
- nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/M);
+ nnQueryObject* query = nnMakeQueryObject(net, M);
- const R test_inputs[M][2] = { { 0., 0. }, { 1., 0. }, { 0., 1. }, { 1., 1. } };
+ const R test_inputs[M][2] = {
+ {0., 0.},
+ {1., 0.},
+ {0., 1.},
+ {1., 1.}
+ };
nnMatrix test_inputs_matrix = nnMatrixMake(M, 2);
nnMatrixInit(&test_inputs_matrix, (const R*)test_inputs);
nnQuery(net, query, &test_inputs_matrix);
- const R expected_outputs[M] = { 0., 1., 1., 0. };
+ const R expected_outputs[M] = {0., 1., 1., 0.};
for (int i = 0; i < M; ++i) {
const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
- printf("\nInput: (%f, %f), Output: %f, Expected: %f\n",
- test_inputs[i][0], test_inputs[i][1], test_output, expected_outputs[i]);
+ printf(
+ "\nInput: (%f, %f), Output: %f, Expected: %f\n", test_inputs[i][0],
+ test_inputs[i][1], test_output, expected_outputs[i]);
}
for (int i = 0; i < M; ++i) {
const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0);
--
cgit v1.2.3