From 653e98e029a0d0f110b0ac599e50406060bb0f87 Mon Sep 17 00:00:00 2001 From: 3gg <3gg@shellblade.net> Date: Sat, 16 Dec 2023 10:21:16 -0800 Subject: Decouple activations from linear layer. --- src/lib/test/neuralnet_test.c | 103 +++++++++++++-------- .../test/train_linear_perceptron_non_origin_test.c | 46 ++++----- src/lib/test/train_linear_perceptron_test.c | 44 ++++----- src/lib/test/train_sigmoid_test.c | 46 ++++----- src/lib/test/train_xor_test.c | 55 +++++++---- 5 files changed, 169 insertions(+), 125 deletions(-) (limited to 'src/lib/test') diff --git a/src/lib/test/neuralnet_test.c b/src/lib/test/neuralnet_test.c index 14d9438..0f8d7b8 100644 --- a/src/lib/test/neuralnet_test.c +++ b/src/lib/test/neuralnet_test.c @@ -1,8 +1,8 @@ #include -#include #include "activation.h" #include "neuralnet_impl.h" +#include #include "test.h" #include "test_util.h" @@ -10,23 +10,31 @@ #include TEST_CASE(neuralnet_perceptron_test) { - const int num_layers = 1; - const int layer_sizes[] = { 1, 1 }; - const nnActivation layer_activations[] = { nnSigmoid }; - const R weights[] = { 0.3 }; + const int num_layers = 2; + const int input_size = 1; + const R weights[] = {0.3}; + const R biases[] = {0.0}; + const nnLayer layers[] = { + {.type = nnLinear, + .linear = + {.weights = nnMatrixFromArray(1, 1, weights), + .biases = nnMatrixFromArray(1, 1, biases)}}, + {.type = nnSigmoid}, + }; - nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations); + nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size); assert(net); - nnSetWeights(net, weights); - nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1); + nnQueryObject* query = nnMakeQueryObject(net, 1); - const R input[] = { 0.9 }; - R output[1]; + const R input[] = {0.9}; + R output[1]; nnQueryArray(net, query, input, output); const R expected_output = sigmoid(input[0] * weights[0]); - printf("\nOutput: %f, Expected: %f\n", output[0], expected_output); + printf( + "\n[neuralnet_perceptron_test] Output: %f, Expected: %f\n", output[0], + expected_output); TEST_TRUE(double_eq(output[0], expected_output, EPS)); nnDeleteQueryObject(&query); @@ -34,53 +42,66 @@ TEST_CASE(neuralnet_perceptron_test) { } TEST_CASE(neuralnet_xor_test) { - const int num_layers = 2; - const int layer_sizes[] = { 2, 2, 1 }; - const nnActivation layer_activations[] = { nnRelu, nnIdentity }; - const R weights[] = { - 1, 1, 1, 1, // First (hidden) layer. - 1, -2 // Second (output) layer. - }; - const R biases[] = { - 0, -1, // First (hidden) layer. - 0 // Second (output) layer. + // First (hidden) layer. + const R weights0[] = {1, 1, 1, 1}; + const R biases0[] = {0, -1}; + // Second (output) layer. + const R weights1[] = {1, -2}; + const R biases1[] = {0}; + // Network. + const int num_layers = 3; + const int input_size = 2; + const nnLayer layers[] = { + {.type = nnLinear, + .linear = + {.weights = nnMatrixFromArray(2, 2, weights0), + .biases = nnMatrixFromArray(1, 2, biases0)}}, + {.type = nnRelu}, + {.type = nnLinear, + .linear = + {.weights = nnMatrixFromArray(2, 1, weights1), + .biases = nnMatrixFromArray(1, 1, biases1)}}, }; - nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations); + nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size); assert(net); - nnSetWeights(net, weights); - nnSetBiases(net, biases); // First layer weights. - TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 0), 1); - TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 1), 1); - TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 2), 1); - TEST_EQUAL(nnMatrixAt(&net->weights[0], 0, 3), 1); - // Second layer weights. - TEST_EQUAL(nnMatrixAt(&net->weights[1], 0, 0), 1); - TEST_EQUAL(nnMatrixAt(&net->weights[1], 0, 1), -2); + TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 0), 1); + TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 1), 1); + TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 2), 1); + TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.weights, 0, 3), 1); + // Second linear layer (third layer) weights. + TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.weights, 0, 0), 1); + TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.weights, 0, 1), -2); // First layer biases. - TEST_EQUAL(nnMatrixAt(&net->biases[0], 0, 0), 0); - TEST_EQUAL(nnMatrixAt(&net->biases[0], 0, 1), -1); - // Second layer biases. - TEST_EQUAL(nnMatrixAt(&net->biases[1], 0, 0), 0); + TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.biases, 0, 0), 0); + TEST_EQUAL(nnMatrixAt(&net->layers[0].linear.biases, 0, 1), -1); + // Second linear layer (third layer) biases. + TEST_EQUAL(nnMatrixAt(&net->layers[2].linear.biases, 0, 0), 0); // Test. - #define M 4 +#define M 4 - nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/M); + nnQueryObject* query = nnMakeQueryObject(net, M); - const R test_inputs[M][2] = { { 0., 0. }, { 1., 0. }, { 0., 1. }, { 1., 1. } }; + const R test_inputs[M][2] = { + {0., 0.}, + {1., 0.}, + {0., 1.}, + {1., 1.} + }; nnMatrix test_inputs_matrix = nnMatrixMake(M, 2); nnMatrixInit(&test_inputs_matrix, (const R*)test_inputs); nnQuery(net, query, &test_inputs_matrix); - const R expected_outputs[M] = { 0., 1., 1., 0. }; + const R expected_outputs[M] = {0., 1., 1., 0.}; for (int i = 0; i < M; ++i) { const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0); - printf("\nInput: (%f, %f), Output: %f, Expected: %f\n", - test_inputs[i][0], test_inputs[i][1], test_output, expected_outputs[i]); + printf( + "\nInput: (%f, %f), Output: %f, Expected: %f\n", test_inputs[i][0], + test_inputs[i][1], test_output, expected_outputs[i]); } for (int i = 0; i < M; ++i) { const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0); diff --git a/src/lib/test/train_linear_perceptron_non_origin_test.c b/src/lib/test/train_linear_perceptron_non_origin_test.c index 5a320ac..40a42e0 100644 --- a/src/lib/test/train_linear_perceptron_non_origin_test.c +++ b/src/lib/test/train_linear_perceptron_non_origin_test.c @@ -1,9 +1,8 @@ #include +#include "neuralnet_impl.h" #include #include -#include "activation.h" -#include "neuralnet_impl.h" #include "test.h" #include "test_util.h" @@ -11,19 +10,21 @@ #include TEST_CASE(neuralnet_train_linear_perceptron_non_origin_test) { - const int num_layers = 1; - const int layer_sizes[] = { 1, 1 }; - const nnActivation layer_activations[] = { nnIdentity }; + const int num_layers = 1; + const int input_size = 1; + const nnLayer layers[] = { + {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}} + }; - nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations); + nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size); assert(net); - // Train. +// Train. - // Try to learn the Y = 2X + 1 line. - #define N 2 - const R inputs[N] = { 0., 1. }; - const R targets[N] = { 1., 3. }; +// Try to learn the Y = 2X + 1 line. +#define N 2 + const R inputs[N] = {0., 1.}; + const R targets[N] = {1., 3.}; nnMatrix inputs_matrix = nnMatrixMake(N, 1); nnMatrix targets_matrix = nnMatrixMake(N, 1); @@ -31,31 +32,32 @@ TEST_CASE(neuralnet_train_linear_perceptron_non_origin_test) { nnMatrixInit(&targets_matrix, targets); nnTrainingParams params = { - .learning_rate = 0.7, - .max_iterations = 20, - .seed = 0, - .weight_init = nnWeightInit01, - .debug = false, + .learning_rate = 0.7, + .max_iterations = 20, + .seed = 0, + .weight_init = nnWeightInit01, + .debug = false, }; nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms); - const R weight = nnMatrixAt(&net->weights[0], 0, 0); + const R weight = nnMatrixAt(&net->layers[0].linear.weights, 0, 0); const R expected_weight = 2.0; - printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight); + printf( + "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight); TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS)); - const R bias = nnMatrixAt(&net->biases[0], 0, 0); + const R bias = nnMatrixAt(&net->layers[0].linear.biases, 0, 0); const R expected_bias = 1.0; printf("Trained network bias: %f, Expected: %f\n", bias, expected_bias); TEST_TRUE(double_eq(bias, expected_bias, WEIGHT_EPS)); // Test. - nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1); + nnQueryObject* query = nnMakeQueryObject(net, 1); - const R test_input[] = { 2.3 }; - R test_output[1]; + const R test_input[] = {2.3}; + R test_output[1]; nnQueryArray(net, query, test_input, test_output); const R expected_output = test_input[0] * expected_weight + expected_bias; diff --git a/src/lib/test/train_linear_perceptron_test.c b/src/lib/test/train_linear_perceptron_test.c index 2b1336d..667643b 100644 --- a/src/lib/test/train_linear_perceptron_test.c +++ b/src/lib/test/train_linear_perceptron_test.c @@ -1,9 +1,8 @@ #include +#include "neuralnet_impl.h" #include #include -#include "activation.h" -#include "neuralnet_impl.h" #include "test.h" #include "test_util.h" @@ -11,19 +10,21 @@ #include TEST_CASE(neuralnet_train_linear_perceptron_test) { - const int num_layers = 1; - const int layer_sizes[] = { 1, 1 }; - const nnActivation layer_activations[] = { nnIdentity }; + const int num_layers = 1; + const int input_size = 1; + const nnLayer layers[] = { + {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}} + }; - nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations); + nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size); assert(net); - // Train. +// Train. - // Try to learn the Y=X line. - #define N 2 - const R inputs[N] = { 0., 1. }; - const R targets[N] = { 0., 1. }; +// Try to learn the Y=X line. +#define N 2 + const R inputs[N] = {0., 1.}; + const R targets[N] = {0., 1.}; nnMatrix inputs_matrix = nnMatrixMake(N, 1); nnMatrix targets_matrix = nnMatrixMake(N, 1); @@ -31,26 +32,27 @@ TEST_CASE(neuralnet_train_linear_perceptron_test) { nnMatrixInit(&targets_matrix, targets); nnTrainingParams params = { - .learning_rate = 0.7, - .max_iterations = 10, - .seed = 0, - .weight_init = nnWeightInit01, - .debug = false, + .learning_rate = 0.7, + .max_iterations = 10, + .seed = 0, + .weight_init = nnWeightInit01, + .debug = false, }; nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms); - const R weight = nnMatrixAt(&net->weights[0], 0, 0); + const R weight = nnMatrixAt(&net->layers[0].linear.weights, 0, 0); const R expected_weight = 1.0; - printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight); + printf( + "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight); TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS)); // Test. - nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1); + nnQueryObject* query = nnMakeQueryObject(net, 1); - const R test_input[] = { 2.3 }; - R test_output[1]; + const R test_input[] = {2.3}; + R test_output[1]; nnQueryArray(net, query, test_input, test_output); const R expected_output = test_input[0]; diff --git a/src/lib/test/train_sigmoid_test.c b/src/lib/test/train_sigmoid_test.c index 588e7ca..39a84b0 100644 --- a/src/lib/test/train_sigmoid_test.c +++ b/src/lib/test/train_sigmoid_test.c @@ -1,9 +1,9 @@ #include -#include -#include #include "activation.h" #include "neuralnet_impl.h" +#include +#include #include "test.h" #include "test_util.h" @@ -11,21 +11,24 @@ #include TEST_CASE(neuralnet_train_sigmoid_test) { - const int num_layers = 1; - const int layer_sizes[] = { 1, 1 }; - const nnActivation layer_activations[] = { nnSigmoid }; + const int num_layers = 2; + const int input_size = 1; + const nnLayer layers[] = { + {.type = nnLinear, .linear = {.input_size = 1, .output_size = 1}}, + {.type = nnSigmoid}, + }; - nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations); + nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size); assert(net); - // Train. +// Train. - // Try to learn the sigmoid function. - #define N 3 +// Try to learn the sigmoid function. +#define N 3 R inputs[N]; R targets[N]; for (int i = 0; i < N; ++i) { - inputs[i] = lerp(-1, +1, (R)i / (R)(N-1)); + inputs[i] = lerp(-1, +1, (R)i / (R)(N - 1)); targets[i] = sigmoid(inputs[i]); } @@ -35,29 +38,30 @@ TEST_CASE(neuralnet_train_sigmoid_test) { nnMatrixInit(&targets_matrix, targets); nnTrainingParams params = { - .learning_rate = 0.9, - .max_iterations = 100, - .seed = 0, - .weight_init = nnWeightInit01, - .debug = false, + .learning_rate = 0.9, + .max_iterations = 100, + .seed = 0, + .weight_init = nnWeightInit01, + .debug = false, }; nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms); - const R weight = nnMatrixAt(&net->weights[0], 0, 0); + const R weight = nnMatrixAt(&net->layers[0].linear.weights, 0, 0); const R expected_weight = 1.0; - printf("\nTrained network weight: %f, Expected: %f\n", weight, expected_weight); + printf( + "\nTrained network weight: %f, Expected: %f\n", weight, expected_weight); TEST_TRUE(double_eq(weight, expected_weight, WEIGHT_EPS)); // Test. - nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/1); + nnQueryObject* query = nnMakeQueryObject(net, 1); - const R test_input[] = { 0.3 }; - R test_output[1]; + const R test_input[] = {0.3}; + R test_output[1]; nnQueryArray(net, query, test_input, test_output); - const R expected_output = 0.574442516811659; // sigmoid(0.3) + const R expected_output = 0.574442516811659; // sigmoid(0.3) printf("Output: %f, Expected: %f\n", test_output[0], expected_output); TEST_TRUE(double_eq(test_output[0], expected_output, OUTPUT_EPS)); diff --git a/src/lib/test/train_xor_test.c b/src/lib/test/train_xor_test.c index 6ddc6e0..78695a3 100644 --- a/src/lib/test/train_xor_test.c +++ b/src/lib/test/train_xor_test.c @@ -1,9 +1,9 @@ #include -#include -#include #include "activation.h" #include "neuralnet_impl.h" +#include +#include #include "test.h" #include "test_util.h" @@ -11,18 +11,27 @@ #include TEST_CASE(neuralnet_train_xor_test) { - const int num_layers = 2; - const int layer_sizes[] = { 2, 2, 1 }; - const nnActivation layer_activations[] = { nnRelu, nnIdentity }; + const int num_layers = 3; + const int input_size = 2; + const nnLayer layers[] = { + {.type = nnLinear, .linear = {.input_size = 2, .output_size = 2}}, + {.type = nnRelu}, + {.type = nnLinear, .linear = {.input_size = 2, .output_size = 1}} + }; - nnNeuralNetwork* net = nnMakeNet(num_layers, layer_sizes, layer_activations); + nnNeuralNetwork* net = nnMakeNet(layers, num_layers, input_size); assert(net); // Train. - #define N 4 - const R inputs[N][2] = { { 0., 0. }, { 0., 1. }, { 1., 0. }, { 1., 1. } }; - const R targets[N] = { 0., 1., 1., 0. }; +#define N 4 + const R inputs[N][2] = { + {0., 0.}, + {0., 1.}, + {1., 0.}, + {1., 1.} + }; + const R targets[N] = {0., 1., 1., 0.}; nnMatrix inputs_matrix = nnMatrixMake(N, 2); nnMatrix targets_matrix = nnMatrixMake(N, 1); @@ -30,31 +39,37 @@ TEST_CASE(neuralnet_train_xor_test) { nnMatrixInit(&targets_matrix, targets); nnTrainingParams params = { - .learning_rate = 0.1, - .max_iterations = 500, - .seed = 0, - .weight_init = nnWeightInit01, - .debug = false, + .learning_rate = 0.1, + .max_iterations = 500, + .seed = 0, + .weight_init = nnWeightInit01, + .debug = false, }; nnTrain(net, &inputs_matrix, &targets_matrix, ¶ms); // Test. - #define M 4 +#define M 4 - nnQueryObject* query = nnMakeQueryObject(net, /*num_inputs=*/M); + nnQueryObject* query = nnMakeQueryObject(net, M); - const R test_inputs[M][2] = { { 0., 0. }, { 1., 0. }, { 0., 1. }, { 1., 1. } }; + const R test_inputs[M][2] = { + {0., 0.}, + {1., 0.}, + {0., 1.}, + {1., 1.} + }; nnMatrix test_inputs_matrix = nnMatrixMake(M, 2); nnMatrixInit(&test_inputs_matrix, (const R*)test_inputs); nnQuery(net, query, &test_inputs_matrix); - const R expected_outputs[M] = { 0., 1., 1., 0. }; + const R expected_outputs[M] = {0., 1., 1., 0.}; for (int i = 0; i < M; ++i) { const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0); - printf("\nInput: (%f, %f), Output: %f, Expected: %f\n", - test_inputs[i][0], test_inputs[i][1], test_output, expected_outputs[i]); + printf( + "\nInput: (%f, %f), Output: %f, Expected: %f\n", test_inputs[i][0], + test_inputs[i][1], test_output, expected_outputs[i]); } for (int i = 0; i < M; ++i) { const R test_output = nnMatrixAt(nnNetOutputs(query), i, 0); -- cgit v1.2.3