Почему моя ИНС на C# не сходится, несмотря на настройку гиперпараметров? ⇐ C#

[Anonymous]

У меня есть ИНС, реализованная с нуля на C#. Однако не сходилось, в чем может быть проблема? Я пробовал разные функции активации, другой набор данных. Набор данных генерируется случайным образом, но ИНС в Python работает с тем же набором данных, поэтому можно обучить нейронную сеть данным.

Код: Выделить всё

private NeuralNetworkData ReLU(NeuralNetworkData x)
{
double[,] res = new double[x.Data.GetLength(0), x.Data.GetLength(1)];
for (int i = 0; i < res.Length; i++)
{
res[i, 0] = MathF.Max(0, (float)x.Data[i,0]);
}
return new NeuralNetworkData( res);
}
private NeuralNetworkData Softmax(NeuralNetworkData x)
{
double[,] res = new double[x.Data.GetLength(0), x.Data.GetLength(1)];
double max = x.ToVector().Max(x => x);
double[] exps = x.ToVector().Select(x => Math.Exp(x - max)).ToArray();
double sumExps = exps.Sum();

for (int i = 0; i < res.Length; i++)
{
res[i, 0] = exps[i] / sumExps;
}
return new NeuralNetworkData(res);
}
private NeuralNetworkData ReLUDerivative(NeuralNetworkData x)
{
double[,] res = new double[x.Data.GetLength(0), x.Data.GetLength(1)];
for (int i = 0; i < res.Length; i++)
{
res[i, 0] = x.Data[i, 0] > 0 ? 1 : 0;
}
return new NeuralNetworkData(res);
}
private void FeedForward(int index, int epoch)
{
if (epoch is 0 && index is 0)
{
Weights1 = Weights1.Transpose();
Weights2 = Weights2.Transpose();
}
Input = Scale(DataFile.Data[index].Values).ToMatrix();
Z1 = (Weights1 * Input) + Bias1;
Hidden = ReLU(Z1);
Z2 = (Weights2 * Hidden) + Bias2;
Output =  Softmax(Z2);

//Console.WriteLine($"N1: {Output[0]}, N2: {Output[1]}, N3: {Output[2]}");
}
private double Error(int index)
{
double localError = 0;
for (int i = 0; i < Output.Data.GetLength(0); i++)
{
localError -= (DataFile.Data[index].Label[i] * Math.Log(Output.Data[i, 0] + 1e-9));
}

return localError;
}
private NeuralNetworkData ElementWiseMultiply(NeuralNetworkData first, NeuralNetworkData second)
{
if (first.Data.GetLength(0) == second.Data.GetLength(0))
{
double[] res = new double[first.Data.GetLength(0)];
for (int i = 0; i < res.Length; i++)
{
res[i] = first.Data[i, 0] * second.Data[i, 0];
}
return res.ToMatrix();
}
else
{
throw new Exception("Two vectors are not the same length!");
}
}
private void Backpropagate(int index)
{
Delta2 = Output - DataFile.Data[index].Label.ToMatrix();
Output_Gradient = Delta2 * Hidden.Transpose();
Bias2_Gradient = Delta2;

Delta1 = ElementWiseMultiply(Weights2.Transpose() * Delta2 , ReLUDerivative(Z1));

Hidden_Gradient = Delta1 * Input.Transpose();
Bias1_Gradient = Delta1;

Weights2 = Weights2 - (LearningRate * Output_Gradient);
Bias2 = Bias2 - (LearningRate * Bias2_Gradient);

Weights1 = Weights1 - (LearningRate * Hidden_Gradient);
Bias1 = Bias1 - (LearningRate * Bias1_Gradient);
}
private double[] Scale(double[] data)
{
double[] res = new double[data.Length];
double sum = data.Sum();
for (int i = 0; i < res.Length; i++)
{
res[i] = data[i] / sum;
}
return res;
}
}
public static class DoubleExtensions
{
public static bool Any(this double[,] matrix, Func func)
{
for (int i = 0; i < matrix.GetLength(0); i++)
{
for (int j = 0; j < matrix.GetLength(1); j++)
{
if (!func.Invoke(matrix[i, j]))
{
return true;
}
}
}
return false;
}
public static bool All(this double[,] matrix, Func func)
{
List isTrue = new List();
for (int i = 0; i < matrix.GetLength(0); i++)
{
for (int j = 0; j < matrix.GetLength(1);  j++)
{
if (func.Invoke(matrix[i, j]))
{
isTrue.Add(true);
}
else
{
isTrue.Add(false);
}
}
}
return isTrue.All(x => x is true);
}
public static NeuralNetworkData ToMatrix(this double[] vector)
{
if (vector.Any(x => x != 0))
{
double[,] res = new double[vector.Length,1];
for (int i = 0; i < res.GetLength(0); i++)
{
res[i, 0] = vector[i];
}
return new NeuralNetworkData(res);
}
else
{
throw new Exception("No input.");
}
}
public static NeuralNetworkData Transpose(this NeuralNetworkData matrix)
{
if (matrix != null)
{
double[,] res = new double[matrix.Data.GetLength(1), matrix.Data.GetLength(0)];
for (int i = 0; i < matrix.Data.GetLength(0); i++)
{
for (int j = 0; j < matrix.Data.GetLength(1); j++)
{
res[j, i] = matrix.Data[i, j];
}
}
return new NeuralNetworkData(res);
}
else
{
throw new Exception("No input provided!");
}
}
public static double[] ToVector(this NeuralNetworkData matrix)
{
if (matrix.Data.GetLength(1) == 1)
{
double[] res = new double[matrix.Data.GetLength(0)];
for (int i = 0; i < res.Length; i++)
{
res[i] = matrix.Data[i, 0];
}

return (res);
}
else
{
throw new Exception("Matrix too big!");
}
}
public static double[] Multiply(this double[] first, double second)
{
double[] res = new double[first.Length];
for (int i = 0; i < res.Length; i++)
{
res[i] = first[i] * second;
}
return res;
}
}

Вот данные NeuralNetworkData:

Код: Выделить всё

public class NeuralNetworkData
{

private void FillMatrix()
{
for (int i = 0; i < Data.GetLength(0); i++)
{
for (int j = 0; j < Data.GetLength(1); j++)
{
Data[i, j] = rnd.NextDouble() * 0.01;
}
}
}
public static NeuralNetworkData operator *(NeuralNetworkData first, NeuralNetworkData second)
{
if (first.Data.GetLength(1) != second.Data.GetLength(0))
throw new Exception("Matrices cannot be multiplied!");

double[,] res = new double[first.Data.GetLength(0), second.Data.GetLength(1)];

for (int i = 0; i < res.GetLength(0); i++)
{
for (int j = 0; j < res.GetLength(1); j++)
{
double sum = 0;
for (int k = 0; k < first.Data.GetLength(1); k++)
{
sum += first.Data[i, k] * second.Data[k, j];
}
res[i, j] = sum;
}
}

return new NeuralNetworkData(res);
}
public static NeuralNetworkData operator *(double first, NeuralNetworkData second)
{
double[,] res = new double[second.Data.GetLength(0), second.Data.GetLength(1)];
for (int i = 0; i < second.Data.GetLength(0); i++)
{
for (int j = 0; j < second.Data.GetLength(1); j++)
{
res[i, j] = second.Data[i, j] * first;
}
}
return new NeuralNetworkData(res);
}
public static NeuralNetworkData operator +(NeuralNetworkData first, NeuralNetworkData second)
{
if (first.Data.GetLength(1) == second.Data.GetLength(1) &&
first.Data.GetLength(0) == second.Data.GetLength(0))
{
double[,] res = new double[first.Data.GetLength(0), second.Data.GetLength(1)];
for (int i = 0; i < res.GetLength(0); i++)
{
for (int j = 0; j < res.GetLength(1);  j++)
{
res[i, j] = first.Data[i, j] + second.Data[i, j];
}
}
return new NeuralNetworkData(res);
}
else
{
throw new Exception("Not the same length.");
}
}
public static NeuralNetworkData operator -(NeuralNetworkData first, NeuralNetworkData second)
{
if (first.Data.GetLength(1) == second.Data.GetLength(1) &&
first.Data.GetLength(0) == second.Data.GetLength(0))
{
double[,] res = new double[first.Data.GetLength(0), second.Data.GetLength(1)];
for (int i = 0; i < res.GetLength(0); i++)
{
for (int j = 0; j < res.GetLength(1); j++)
{
res[i, j] = first.Data[i, j] - second.Data[i, j];
}
}
return new NeuralNetworkData(res);
}
else
{
throw new Exception("Not the same length.");
}
}
}

Заранее спасибо за все советы.
Я пробовал разные скорости обучения, разное количество нейронов в скрытом слое. К сожалению, ничего не помогло.

Подробнее здесь: https://stackoverflow.com/questions/785 ... parameters