I keep receiving the same output for each record when I run it on the test data. I've tried numerous things...any suggestions? I've worked on this for almost twelve hours straight, not counting previous work. I'm hoping one of you guys/gals who know whats going on can point me in the right direction.
#include <iostream>
#include <fstream>
#include <string>
#include <sstream>
#include <vector>
#include <time.h>
#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <algorithm>
#include <functional>
#include <ctime>
#include <cstdlib>
/*
* Backpropagating Multilayered Feedforward Neural Network
* With code assistance by Tejpal Chhabra
* Written by:
* Zachary Bush(s0907933)
* Jesse Bordoe(s0907715)
*/
class CBackProp{
//output of each neuron
double **out;
//3-D array to store weights for each neuron
double ***weight;
//delta error value for each neuron
double **delta;
//no of layers in net including input layer
int numl;
//array of numl elements to store size of each layer
int *lsize;
//learning rate
double beta;
//momentum
double alpha;
//storage for weight-change made in previous epoch
double ***prevDwt;
//sigmoid function
double sigmoid(double in);
public:
~CBackProp();
//initializes and allocates memory
CBackProp(int nl,int *sz,double b,double a);
//backpropogates error for one set of input
void bpgt(double *in,double *tgt);
//feed forwards activations for one set of inputs
void ffwd(double *in);
//returns mean square error of the net
double mse(double *tgt) const;
//returns i'th output of the net
double Out(int i) const;
//get weight values
double getWeight(int i, int j, int k);
};
//get weight values
double CBackProp::getWeight(int i, int j, int k)
{
return weight[i][j][k];
}
// initializes and allocates memory on heap
CBackProp::CBackProp(int nl,int *sz,double b,double a):beta(b),alpha(a)
{
// set no of layers and their sizes
numl=nl;
lsize=new int[numl];
for(int i=0;i<numl;i++){
lsize[i]=sz[i];
}
// allocate memory for output of each neuron
out = new double*[numl];
for(int i=0;i<numl;i++){
out[i]=new double[lsize[i]];
}
// allocate memory for delta
delta = new double*[numl];
for(int i=1;i<numl;i++){
delta[i]=new double[lsize[i]];
}
// allocate memory for weights
weight = new double**[numl];
for(int i=1;i<numl;i++){
weight[i]=new double*[lsize[i]];
}
for(int i=1;i<numl;i++){
for(int j=0;j<lsize[i];j++){
weight[i][j]=new double[lsize[i-1]+1];
}
}
// allocate memory for previous weights
prevDwt = new double**[numl];
for(int i=1;i<numl;i++){
prevDwt[i]=new double*[lsize[i]];
}
for(int i=1;i<numl;i++){
for(int j=0;j<lsize[i];j++){
prevDwt[i][j]=new double[lsize[i-1]+1];
}
}
// seed and assign random weights
srand((unsigned)(time(NULL)));
for(int i=1;i<numl;i++)
for(int j=0;j<lsize[i];j++)
for(int k=0;k<lsize[i-1]+1;k++)
weight[i][j][k]=(double)(rand())/(RAND_MAX/2) - 1;//32767
// initialize previous weights to 0 for first iteration
for(int i=1;i<numl;i++)
for(int j=0;j<lsize[i];j++)
for(int k=0;k<lsize[i-1]+1;k++)
prevDwt[i][j][k]=(double)0.0;
}
CBackProp::~CBackProp()
{
// free out
for(int i=0;i<numl;i++)
delete[] out[i];
delete[] out;
// free delta
for(int i=1;i<numl;i++)
delete[] delta[i];
delete[] delta;
// free weight
for(int i=1;i<numl;i++)
for(int j=0;j<lsize[i];j++)
delete[] weight[i][j];
for(int i=1;i<numl;i++)
delete[] weight[i];
delete[] weight;
// free prevDwt
for(int i=1;i<numl;i++)
for(int j=0;j<lsize[i];j++)
delete[] prevDwt[i][j];
for(int i=1;i<numl;i++)
delete[] prevDwt[i];
delete[] prevDwt;
// free layer info
delete[] lsize;
}
// sigmoid function
double CBackProp::sigmoid(double in)
{
return (double)(1/(1+exp(-in)));
}
// mean square error
double CBackProp::mse(double *tgt) const
{
double mse=0;
for(int i=0;i<lsize[numl-1];i++){
mse+=(tgt[i]-out[numl-1][i])*(tgt[i]-out[numl-1][i]);
}
return mse/2;
}
// returns i'th output of the net
double CBackProp::Out(int i) const
{
return out[numl-1][i];
}
// feed forward one set of input
void CBackProp::ffwd(double *in)
{
double sum;
// assign content to input layer
for(int i=0;i<lsize[0];i++)
out[0][i]=in[i]; // output_from_neuron(i,j) Jth neuron in Ith Layer
// assign output(activation) value
// to each neuron usng sigmoid func
for(int i=1;i<numl;i++){ // For each layer
for(int j=0;j<lsize[i];j++){ // For each neuron in current layer
sum=0.0;
for(int k=0;k<lsize[i-1];k++){ // For input from each neuron in preceeding layer
sum+= out[i-1][k]*weight[i][j][k]; // Apply weight to inputs and add to sum
}
sum+=weight[i][j][lsize[i-1]]; // Apply bias
out[i][j]=sigmoid(sum); // Apply sigmoid function
}
}
}
// backpropogate errors from output
// layer uptill the first hidden layer
void CBackProp::bpgt(double *in,double *tgt)
{
double sum;
// update output values for each neuron
ffwd(in);
// find delta for output layer
for(int i=0;i<lsize[numl-1];i++){
delta[numl-1][i]=out[numl-1][i]*
(1-out[numl-1][i])*(tgt[i]-out[numl-1][i]);
}
// find delta for hidden layers
for(int i=numl-2;i>0;i--){
for(int j=0;j<lsize[i];j++){
sum=0.0;
for(int k=0;k<lsize[i+1];k++){
sum+=delta[i+1][k]*weight[i+1][k][j];
}
delta[i][j]=out[i][j]*(1-out[i][j])*sum;
}
}
// apply momentum ( does nothing if alpha=0 )
for(int i=1;i<numl;i++){
for(int j=0;j<lsize[i];j++){
for(int k=0;k<lsize[i-1];k++){
weight[i][j][k]+=alpha*prevDwt[i][j][k];
}
weight[i][j][lsize[i-1]]+=alpha*prevDwt[i][j][lsize[i-1]];
}
}
// adjust weights usng steepest descent
for(int i=1;i<numl;i++){
for(int j=0;j<lsize[i];j++){
for(int k=0;k<lsize[i-1];k++){
prevDwt[i][j][k]=beta*delta[i][j]*out[i-1][k];
weight[i][j][k]+=prevDwt[i][j][k];
}
prevDwt[i][j][lsize[i-1]]=beta*delta[i][j];
weight[i][j][lsize[i-1]]+=prevDwt[i][j][lsize[i-1]];
}
}
}
using namespace std;
int stringToInt(string convert);
int main(){
string answers;
string dataset;
string directory;
string answersDirectory;
string datasetDirectory;
size_t found;
string currentFile=__FILE__;
string stringHolder;
int intHolder;
vector<vector<double> > input;
vector<vector<double> > testData;
int records;
int features;
int y=0;
int z=0;
int numLayers=3;
int layerSize[3]={3,6,1}; //prev 3
double beta = 0.3;
double alpha = 0.1;
double thresh = 0.00001;
double result = 0;
int num_iter = 1000; //prev 1000
int i=0;
int correct=0;
int recordCount=0;
double accuracy;
vector<int> randomize;
//randomize
srand(time(NULL));
answers="answers.dat";
dataset="dataset.dat";
found=currentFile.find_last_of("\\");
directory=currentFile.substr(0,found);
directory+="\\data\\";
answersDirectory=directory + answers;
datasetDirectory=directory + dataset;
ifstream openAnswers;
ifstream openDataset;
openAnswers.open(answersDirectory.c_str());
openDataset.open(datasetDirectory.c_str());
if(openAnswers.is_open()&&openDataset.is_open())
{
getline(openDataset,stringHolder,' ');
intHolder=stringToInt(stringHolder);
records=intHolder;
getline(openDataset,stringHolder,' ');
intHolder=stringToInt(stringHolder);
features=intHolder;
layerSize[0]=records;
input.resize(records*2);
testData.resize(records*2);
for(int x=0; x < (records); x++)
{
input[x].resize((features+1)*2);
testData[x].resize((features+1)*2);
cout << "Resizing vector to fit records." << endl;
}
getline(openAnswers,stringHolder); //To remove 'records' from the top of answers.dat
while(!openAnswers.eof() && !openDataset.eof() && recordCount<records)
{
if(openDataset.peek()=='\n')
{
getline(openDataset,stringHolder);
getline(openAnswers,stringHolder);
if(stringHolder=="ad")
input[y][z]=1;
else
input[y][z]=0;
y++;
z=0;
cout << "new record added, finished record: " << recordCount <<endl;
recordCount++;
}
else
{
getline(openDataset,stringHolder,' ');
intHolder=stringToInt(stringHolder);
input[y][z]=intHolder;
z++;
cout << "filling vector with values. record #: " << recordCount << endl;
}
}
cout << "Files opened successfully" << endl;
}
else
cout << "Files failed opening." << endl;
for(int i=0;i<records;i++)
randomize.push_back(i);
random_shuffle(randomize.begin(),randomize.end());
CBackProp *bp = new CBackProp(numLayers, layerSize, beta, alpha);
cout<< endl << "Now training the network...." << endl;
for (int i=0; i<num_iter ; i++)
{
double* ptr;
ptr=(double*) &input[i%records];
bp->bpgt(ptr, &input[i%records][features]);
if( bp->mse(&input[i%records][features]) < thresh) {
cout << endl << "Network Trained. threshold value achieved in " << i << "iterations." << endl;
cout << "MSE: " << bp->mse(&input[i%records][features])
<< endl << endl;
break;
}
if ( i%(num_iter/10) == 0 )
cout<< endl << "MSE: " << bp->mse(&input[i%records][features])
<< "... Training..." << endl;
}
if ( i == num_iter )
cout << endl << i << " iterations completed..." << "MSE: " << bp->mse(&input[(i-1)%records][features]) << endl;
cout<< "Now using the trained network to make predctions on test input...." << endl << endl;
double* ptr2;
//ptr2=(double*) &testData[i];
/*recordCount=0;
for(int i=0;i<records;i++)
randomize.push_back(i);
random_shuffle(randomize.begin(),randomize.end());
while(!randomize.empty())
{
i=randomize.back();
randomize.pop_back();
ptr2=(double*) &testData[i];
bp->ffwd(ptr2);
cout << recordCount << " " << bp->Out(0) << endl;
recordCount++;
}
*/
for(int i=0;i<records;i++)
randomize.push_back(i);
random_shuffle(randomize.begin(),randomize.end());
ofstream best;
best.open("data//best.nn");
if(best.is_open())
{
ifstream check;
check.open(answersDirectory.c_str());
if(check.is_open())
{
cout << "Files opened successfully" << endl;
getline(check,stringHolder); //To remove 'records' from the top of answers.dat
/*while(!check.eof() && recordCount<records)
{
getline(check,stringHolder);
if(stringHolder=="ad" && bp->Out(0) >.5)
correct++;
else if(stringHolder=="nonad" && bp->Out(0) <.5)
correct++;
recordCount++;
}
accuracy=correct/recordCount;
best << accuracy << endl;
cout << "Files opened successfully" << endl;*/
getline(check,stringHolder); //To remove 'records' from the top of answers.dat
for (int i = 0 ; i < records ; i++ )
{
ptr2=(double*) &testData[i];
bp->ffwd(ptr2);
result=bp->Out(0);
cout << i << " " << result << endl;
getline(check,stringHolder);
if(stringHolder=="ad" && result >.5)
correct++;
else if(stringHolder=="nonad" && result <.5)
correct++;
}
accuracy=correct/records;
best << accuracy << endl;
best << numLayers-2 << endl;
best << "I " << records << " H " << layerSize[1] << endl;
for(int k=0; k<layerSize[1]; k++)
{
for(int j=0;j<records;j++)
//best << bp->getWeight(1,j,k) << " ";
best << bp->getWeight(1,k,j) << " ";
}
best << endl << "H " << layerSize[1] << " O 1" << endl;
for(int j=0; j<layerSize[2]; j++)
for(int k=0;k<layerSize[1]; k++)
best << bp->getWeight(2,j,k) << " ";
}
else
cout << "answers failed to open" << endl;
best.close();
}
else
cout << "best failed to write. " << endl;
return 0;
}
int stringToInt(string convert)
{
int returnInt;
istringstream integer(convert);
integer >> returnInt;
return returnInt;
}
Small names file:
35
ad
nonad
ad
ad
nonad
ad
nonad
ad
ad
nonad
ad
nonad
nonad
nonad
nonad
ad
ad
ad
nonad
nonad
ad
nonad
ad
nonad
nonad
ad
nonad
nonad
ad
nonad
nonad
ad
ad
nonad
nonad
small dataset:
35 50
1 0 0 1 0 1 0 0 1 0 0 1 1 1 1 0 0 1 0 1 1 1 0 1 1 0 1 0 0 1 1 1 0 0 1 0 0 0 1 1 0 1 0 1 1 1 0 0 0 0
0 0 0 0 1 1 1 1 0 0 0 1 1 0 1 1 0 1 1 1 0 1 1 1 0 1 0 1 1 1 1 0 1 1 1 0 0 1 0 1 1 0 1 0 1 0 1 0 1 1
1 0 0 0 1 1 1 0 1 1 0 1 1 0 1 1 1 0 1 1 0 1 0 1 0 1 1 0 0 1 1 0 0 1 0 0 1 0 1 0 0 1 0 0 1 1 1 1 0 1
0 1 0 1 0 0 1 1 0 0 1 1 1 1 0 0 0 1 0 0 0 0 0 0 1 0 0 1 1 1 0 0 0 0 0 1 1 1 0 1 1 1 1 1 1 0 1 1 0 1
1 1 1 0 1 1 1 0 0 1 0 0 0 0 1 1 1 1 1 1 0 0 1 0 0 0 0 0 1 0 1 0 1 0 0 1 0 0 0 0 0 1 1 0 1 0 1 1 1 1
0 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 1 0 0 0 1 1 0 1 0 1 1 1 1 0 1 1 1 1 1 0 0 0 1 1 1 0 0 1 1 0 0 0 1 1
0 1 0 0 1 0 1 0 0 0 0 0 0 1 0 1 0 0 0 0 1 0 0 1 0 1 1 1 1 0 1 0 1 1 0 0 0 1 1 1 0 0 0 1 0 1 0 1 0 1
0 1 0 1 0 0 0 1 0 0 0 1 0 0 0 0 0 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 0 1 0 1 0 0 1 0 0 0 0 0 0 0 1 0 0
0 1 1 0 1 0 1 0 0 1 1 0 1 0 0 0 0 1 1 1 0 0 0 1 1 1 0 1 0 0 0 0 1 1 0 1 1 1 1 0 1 0 1 1 0 1 1 0 0 0
0 1 0 1 0 0 1 0 1 0 1 0 1 1 1 0 1 1 0 0 1 0 1 0 0 0 0 0 1 1 0 0 0 1 0 1 1 1 0 1 0 1 1 0 1 0 1 0 0 0
0 0 1 1 1 0 0 0 0 0 1 1 1 0 1 0 1 1 1 0 1 1 0 1 0 1 1 0 1 0 1 0 1 1 1 1 1 0 0 0 1 1 1 0 1 1 1 0 0 1
1 1 0 0 0 0 0 1 1 1 0 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 0 0 1 0 1 1 1 0 1 0 0 0 0 1 1 1 0 1 1 1 0
0 1 0 1 1 1 0 1 1 1 1 1 0 0 0 1 0 1 0 1 0 0 1 0 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1 0 1 0 0 0 0 0 0 0 0
1 1 0 1 0 1 1 0 0 1 0 0 0 1 0 1 0 1 0 1 0 1 1 1 1 1 0 1 1 0 0 1 1 0 1 0 1 0 0 1 0 0 0 1 0 0 1 1 1 0
1 0 0 0 1 1 0 0 0 0 0 0 1 1 0 0 1 1 0 1 0 1 1 1 0 1 1 1 1 0 0 1 1 1 1 1 0 0 0 0 1 0 0 1 0 1 1 1 0 1
0 1 0 0 0 1 0 0 0 0 0 1 1 0 0 0 0 1 0 0 1 0 0 1 1 0 1 0 1 0 1 1 0 0 1 1 0 1 1 0 0 1 1 1 0 0 1 0 1 1
0 1 1 0 0 1 1 0 1 0 0 0 0 1 1 1 0 1 0 1 1 1 0 0 1 1 0 1 0 1 0 0 1 0 0 0 1 0 1 1 1 1 1 1 1 0 1 1 0 0
1 0 1 1 0 1 0 1 1 0 0 0 1 1 0 1 1 1 1 1 0 1 0 1 0 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 1 0 0 1 0 1 1 0 0 0
1 0 1 1 0 0 0 1 0 1 0 1 0 0 1 1 0 0 1 0 1 0 1 0 0 1 0 1 0 0 1 0 1 1 0 0 1 1 1 0 0 1 0 0 1 1 0 0 1 0
1 1 0 0 1 0 1 0 0 1 1 1 1 1 0 1 0 0 1 0 1 0 0 1 0 1 0 1 0 1 1 0 0 1 1 1 0 0 1 1 0 0 1 0 0 1 0 1 1 0
1 0 0 1 1 0 0 1 1 0 0 0 1 0 1 0 1 1 1 0 1 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 1 0 0 0 1 1 1 1 1
1 1 0 1 1 1 0 0 0 1 1 0 1 1 0 0 0 1 1 0 1 1 1 0 0 0 0 1 0 1 0 1 1 0 1 1 0 0 0 1 0 0 0 1 1 0 1 1 0 0
1 0 0 0 0 0 0 0 0 1 1 0 0 0 1 0 1 1 1 1 0 1 0 0 0 0 1 1 1 1 0 1 1 1 1 0 0 1 1 0 1 0 0 1 1 1 0 1 0 0
1 1 1 1 0 1 1 1 1 0 0 0 0 0 1 0 1 1 0 1 0 0 1 0 1 1 1 0 0 0 0 1 1 0 1 0 1 1 1 1 0 0 1 1 1 0 1 1 0 0
0 0 0 0 0 1 1 0 0 0 1 1 1 0 1 1 0 1 0 0 1 1 0 1 1 1 0 1 0 1 1 1 0 1 0 1 0 0 1 0 0 0 1 0 1 0 0 0 1 1
1 0 0 1 1 0 1 1 0 0 0 1 0 1 0 0 0 0 1 1 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 1 0 1 1 0 0 0
1 0 1 1 1 0 1 0 0 1 1 0 0 0 1 0 0 0 0 0 1 1 0 1 0 0 1 0 1 0 0 0 1 1 0 0 0 0 1 0 0 1 1 1 0 0 0 0 1 1
1 0 1 0 1 0 1 0 0 0 1 0 1 1 1 0 0 0 1 1 1 1 0 1 0 1 1 1 1 0 1 1 0 0 0 1 1 1 0 1 1 1 1 1 0 0 1 1 0 0
0 1 1 1 0 0 1 0 0 0 0 1 1 0 1 0 1 1 1 0 1 1 1 1 1 0 1 0 1 1 1 0 1 0 0 0 1 0 1 1 1 0 1 1 1 0 1 1 1 0
0 0 0 1 1 0 1 0 1 1 0 1 0 1 1 1 0 1 1 1 0 0 0 0 0 1 0 0 0 1 0 1 0 1 1 1 0 1 1 1 1 0 0 0 1 1 0 1 1 1
0 1 1 1 0 1 1 1 1 1 0 1 1 0 0 0 1 1 0 1 1 1 1 0 0 1 0 0 0 0 0 1 0 1 0 1 1 0 0 1 1 1 1 1 0 1 1 0 0 1
1 1 1 1 0 0 1 0 1 1 1 0 0 0 1 1 0 1 1 0 1 1 0 1 0 0 0 0 0 0 0 0 0 0 1 1 0 1 1 1 1 0 1 1 0 1 1 0 1 0
1 0 0 1 1 0 0 1 1 0 0 1 1 1 1 1 1 1 1 1 1 0 1 1 0 1 1 1 1 1 0 0 1 0 0 0 1 0 1 1 1 1 1 1 0 0 0 0 0 1
1 0 0 0 0 1 1 1 1 1 1 0 1 1 0 1 0 0 1 0 0 1 1 1 1 1 1 0 0 0 0 1 0 0 0 1 0 0 1 0 1 1 1 0 0 0 0 1 1 1
1 1 0 0 1 1 0 1 0 0 1 1 0 1 1 0 0 1 0 0 0 0 0 1 0 1 1 0 0 1 0 1 1 0 0 1 1 0 0 0 1 1 0 0 1 1 0 1 1 1
I have the large datasets too, but they're on the scale of 2000 records and 1500 features...so I don't want to fill up the forums.
Thanks in advance.