Fixed errors and preformance tunning.

I fixed an issue in the cross entropy algorithm when it gave a 0% probability for a given character, causing it to take a log2(0).
Now it adds a small fraction to prevent it from ever trying to take an invalid log.

I also normalized the object file to [0.0, 1.0] which fixed the problem of always choosing the same output.

I also adjusted the learning constant and some training variables to get better accuracy.

hwRecProto2.py

@@ -68,8 +68,19 @@ with open('iamDataset.obj', 'rb') as input:
 ########################################################################################
 # Import tensorflow library and define AAN
 
+import sys
+
+print sys.argv
+
+LEARNING_CONST = 0.05
+TRAIN_CYCLES = 7000
+BATCH_SIZE = 800
+
+# Turn off GPU Warnings/All other warnings
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+
 import tensorflow as tf
-LEARNING_CONST = 0.5
 
 # Create a tensorflow placeholder for an array [nx784] float32's (a.k.a. n MNIST vectors)
 inVect = tf.placeholder(tf.float32, [None, N_INPUT])
@@ -86,7 +97,7 @@ outVectCorr = tf.placeholder(tf.float32, [None, N_OUTPUT])
 # Calculate how incorrect the solutions arrive were
 crossEntropy = tf.reduce_mean(
     -tf.reduce_sum(
-        outVectCorr * tf.log(outVect),
+        outVectCorr * tf.log(outVect + 1e-37),
         # Tells reduce_sum to use the 10-length array, and not the n-length
         reduction_indices=[1]
     )
@@ -97,13 +108,23 @@ trainStep = tf.train.GradientDescentOptimizer(LEARNING_CONST).minimize(crossEntr
 ########################################################################################
 # Define accuracy checking conditions
 
+predicted = tf.argmax(outVect, 1)
+
 # Define formula for determining correctness
 # Highest value in outVect 1st index == highest value in correct outVect 1st index
-predictionCorr = tf.equal(tf.argmax(outVect, 1), tf.argmax(outVectCorr, 1))
+predictionCorr = tf.equal(predicted, tf.argmax(outVectCorr, 1))
 
 # Calculate how accurate the system was
 accuracy = tf.reduce_mean(tf.cast(predictionCorr, tf.float32))
 
+# Print statements for debug
+printOut = tf.Print(outVect, [outVect], "Output Vector: ", summarize=N_OUTPUT*3)
+printWeights = tf.Print(weights, [weights], "Weight Matrix: ")
+printBias = tf.Print(biases, [biases], "Bias Vecotr: ")
+printinVect = tf.Print(inVect, [inVect], "Input Vector: ")
+printEntropy = tf.Print(crossEntropy, [crossEntropy], "Cross Entropy: ")
+printLog2Out = tf.Print(tf.log(outVect + 1e-10), [tf.log(outVect + 1e-10)], "Log2Out: ")
+
 ########################################################################################
 # Run the system
 
@@ -113,12 +134,26 @@ sess = tf.InteractiveSession()
 # Initialize variables
 tf.global_variables_initializer().run()
 
-for _ in range(1000) :
+for _ in range(TRAIN_CYCLES) :
     # Get 100 random digits from training set
-    batchIns, batchOuts = iam.nextBatch(100)
+    batchIns, batchOuts = iam.nextBatch(BATCH_SIZE)
     # Run the training step in the interactive session with the given inputs/outputs
     sess.run(trainStep, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
 
+#batchIns, batchOuts = iam.nextBatch(100)
+#sess.run(printinVect, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
+#sess.run(printWeights, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
+#sess.run(printBias, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
+#sess.run(printOut, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
+#sess.run(printLog2Out, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
+#sess.run(printEntropy, feed_dict={inVect: batchIns, outVectCorr: batchOuts})
+
+
 # Check accuracy
 testIn, testOuts = iam.testSet()
+#for i in range(0, 1):#len(testIn)):
 print(sess.run(accuracy, feed_dict={inVect: testIn, outVectCorr: testOuts}))
+#sess.run(printinVect, feed_dict={inVect: testIn, outVectCorr: testOuts})
+#sess.run(printWeights, feed_dict={inVect: testIn, outVectCorr: testOuts})
+#sess.run(printBias, feed_dict={inVect: testIn, outVectCorr: testOuts})
+#print(sess.run(printOut, feed_dict={inVect: testIn, outVectCorr: testOuts}))

png2Obj.py

@@ -34,7 +34,7 @@ class IAM:
                 img = scipy.ndimage.imread(IMG_TEMPLATE % (x, f), True)
                 img = scipy.misc.imresize(img, 0.03)
                 img = list(itertools.chain.from_iterable(img))
-                print len(img)
+                img = [float(i)/max(img) for i in img]
                 if len(self.test) < (5 * x):
                     self.test.append(Datum(x, img))
                 else: