From 411f66a2540fa17c736116d865e0ceb0cfe5623b Mon Sep 17 00:00:00 2001 From: jeanne Date: Wed, 11 May 2022 09:54:38 -0700 Subject: Initial commit. --- src/lib/include/neuralnet/matrix.h | 111 ++++++++++++++++++++++++++++++++++ src/lib/include/neuralnet/neuralnet.h | 64 ++++++++++++++++++++ src/lib/include/neuralnet/train.h | 42 +++++++++++++ src/lib/include/neuralnet/types.h | 3 + 4 files changed, 220 insertions(+) create mode 100644 src/lib/include/neuralnet/matrix.h create mode 100644 src/lib/include/neuralnet/neuralnet.h create mode 100644 src/lib/include/neuralnet/train.h create mode 100644 src/lib/include/neuralnet/types.h (limited to 'src/lib/include') diff --git a/src/lib/include/neuralnet/matrix.h b/src/lib/include/neuralnet/matrix.h new file mode 100644 index 0000000..9816b81 --- /dev/null +++ b/src/lib/include/neuralnet/matrix.h @@ -0,0 +1,111 @@ +#pragma once + +#include + +#include + +/// NxM matrix. +typedef struct nnMatrix { + int rows; + int cols; + R* values; +} nnMatrix; + +/// Construct a matrix. +nnMatrix nnMatrixMake(int rows, int cols); + +/// Delete a matrix and free its internal memory. +void nnMatrixDel(nnMatrix*); + +/// Move a matrix. +/// +/// |in| is an empty matrix after the move. +/// |out| is a matrix like |in| before the move. +void nnMatrixMove(nnMatrix* in, nnMatrix* out); + +/// Deep-copy a matrix. +void nnMatrixCopy(const nnMatrix* in, nnMatrix* out); + +/// Write the matrix values into an array in a row-major fashion. +void nnMatrixToArray(const nnMatrix* in, R* out); + +/// Write the given row of a matrix into an array. +void nnMatrixRowToArray(const nnMatrix* in, int row, R* out); + +/// Copy a column from a source to a target matrix. +void nnMatrixCopyCol(const nnMatrix* in, nnMatrix* out, int col_in, int col_out); + +/// Mutable borrow of a matrix. +nnMatrix nnMatrixBorrow(nnMatrix* in); + +/// Mutable borrow of a subrange of rows of a matrix. +nnMatrix nnMatrixBorrowRows(nnMatrix* in, int row_start, int num_rows); + +/// Initialize the matrix from an array of values. +/// +/// The array must hold values in a row-major fashion. +void nnMatrixInit(nnMatrix*, const R* values); + +/// Initialize all matrix values to a given constant. +void nnMatrixInitConstant(nnMatrix*, R value); + +/// Multiply two matrices. +void nnMatrixMul(const nnMatrix* left, const nnMatrix* right, nnMatrix* out); + +/// Matrix multiply-add. +/// +/// out = left + (right * scale) +void nnMatrixMulAdd(const nnMatrix* left, const nnMatrix* right, R scale, nnMatrix* out); + +/// Matrix multiply-subtract. +/// +/// out = left - (right * scale) +void nnMatrixMulSub(const nnMatrix* left, const nnMatrix* right, R scale, nnMatrix* out); + +/// Hadamard product of two matrices. +void nnMatrixMulPairs(const nnMatrix* left, const nnMatrix* right, nnMatrix* out); + +/// Add two matrices. +void nnMatrixAdd(const nnMatrix* left, const nnMatrix* right, nnMatrix* out); + +/// Subtract two matrices. +void nnMatrixSub(const nnMatrix* left, const nnMatrix* right, nnMatrix* out); + +/// Adds a row vector to all rows of the matrix. +void nnMatrixAddRow(const nnMatrix* matrix, const nnMatrix* row, nnMatrix* out); + +/// Scale a matrix. +void nnMatrixScale(nnMatrix*, R scale); + +/// Transpose a matrix. +/// |in| must be different than |out|. +void nnMatrixTranspose(const nnMatrix* in, nnMatrix* out); + +/// Threshold the values of a matrix using a greater-than operator. +/// +/// out[x,y] = 1 if in[x,y] > threshold else 0 +void nnMatrixGt(const nnMatrix* in, R threshold, nnMatrix* out); + +/// Return the matrix value at the given row and column. +static inline R nnMatrixAt(const nnMatrix* matrix, int row, int col) { + assert(matrix); + return matrix->values[row * matrix->cols + col]; +} + +/// Set the matrix value at the given row and column. +static inline void nnMatrixSet(nnMatrix* matrix, int row, int col, R value) { + assert(matrix); + matrix->values[row * matrix->cols + col] = value; +} + +/// Return a pointer to the given row in the matrix. +static inline const R* nnMatrixRow(const nnMatrix* matrix, int row) { + assert(matrix); + return &matrix->values[row * matrix->cols]; +} + +/// Return a mutable pointer to the given row in the matrix. +static inline R* nnMatrixRow_mut(nnMatrix* matrix, int row) { + assert(matrix); + return &matrix->values[row * matrix->cols]; +} diff --git a/src/lib/include/neuralnet/neuralnet.h b/src/lib/include/neuralnet/neuralnet.h new file mode 100644 index 0000000..1cf1c53 --- /dev/null +++ b/src/lib/include/neuralnet/neuralnet.h @@ -0,0 +1,64 @@ +#pragma once + +#include + +typedef struct nnMatrix nnMatrix; + +typedef struct nnNeuralNetwork nnNeuralNetwork; +typedef struct nnQueryObject nnQueryObject; + +/// Neuron activation. +typedef enum nnActivation { + nnIdentity, + nnSigmoid, + nnRelu, +} nnActivation; + +/// Create a network. +nnNeuralNetwork* nnMakeNet(int num_layers, const int* layer_sizes, const nnActivation* activations); + +/// Delete the network and free its internal memory. +void nnDeleteNet(nnNeuralNetwork**); + +/// Set the network's weights. +void nnSetWeights(nnNeuralNetwork*, const R* weights); + +/// Set the network's biases. +void nnSetBiases(nnNeuralNetwork*, const R* biases); + +/// Query the network. +/// +/// |input| is a matrix of inputs, one row per input and as many columns as the +/// input's dimension. +/// +/// The query object's output matrix (see nnQueryOutputs()) is a matrix of +/// outputs, one row per output and as many columns as the output's dimension. +void nnQuery(const nnNeuralNetwork*, nnQueryObject*, const nnMatrix* input); + +/// Query the network, array version. +void nnQueryArray(const nnNeuralNetwork*, nnQueryObject*, const R* input, R* output); + +/// Create a query object. +/// +/// The query object holds all the internal memory required to query a network. +/// Query objects allocate all memory up front so that network queries can run +/// without additional memory allocation. +nnQueryObject* nnMakeQueryObject(const nnNeuralNetwork*, int num_inputs); + +/// Delete the query object and free its internal memory. +void nnDeleteQueryObject(nnQueryObject**); + +/// Return the outputs of the query. +const nnMatrix* nnNetOutputs(const nnQueryObject*); + +/// Return the network's input size. +int nnNetInputSize(const nnNeuralNetwork*); + +/// Return the network's output size. +int nnNetOutputSize(const nnNeuralNetwork*); + +/// Return the layer's input size. +int nnLayerInputSize(const nnMatrix* weights); + +/// Return the layer's output size. +int nnLayerOutputSize(const nnMatrix* weights); diff --git a/src/lib/include/neuralnet/train.h b/src/lib/include/neuralnet/train.h new file mode 100644 index 0000000..79f8e7b --- /dev/null +++ b/src/lib/include/neuralnet/train.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +typedef struct nnMatrix nnMatrix; + +/// Weight initialization strategy. +/// +/// Note that regardless of strategy, a layer's weights are scaled by the +/// layer's size. This is to avoid saturation when, e.g., using a sigmoid +/// activation with many inputs. Thus, a (0,1) initialization is really +/// (0,scale), for example. +typedef enum nnWeightInitStrategy { + nnWeightInit01, // (0,1) range. + nnWeightInit11, // (-1,+1) range. + nnWeightInitNormal, // Normal distribution. +} nnWeightInitStrategy; + +/// Network training parameters. +typedef struct nnTrainingParams { + R learning_rate; + int max_iterations; + uint64_t seed; + nnWeightInitStrategy weight_init; + bool debug; +} nnTrainingParams; + +/// Train the network. +/// +/// |inputs| is a matrix of inputs, one row per input and as many columns as +/// the input's dimension. +/// +/// |targets| is a matrix of targets, one row per target and as many columns as +/// the target's dimension. +void nnTrain( + nnNeuralNetwork*, + const nnMatrix* inputs, + const nnMatrix* targets, + const nnTrainingParams*); diff --git a/src/lib/include/neuralnet/types.h b/src/lib/include/neuralnet/types.h new file mode 100644 index 0000000..e8d3942 --- /dev/null +++ b/src/lib/include/neuralnet/types.h @@ -0,0 +1,3 @@ +#pragma once + +typedef double R; -- cgit v1.2.3