modify reddit classifier test

Author: Doha2012

spring-security-oauth/src/main/java/org/baeldung/config/WebConfig.java

@@ -86,7 +86,7 @@ public ScheduledTasks scheduledTasks(OAuth2ProtectedResourceDetails reddit) {
 
     @Bean
     public RedditClassifier redditClassifier() throws IOException {
-        final Resource file = new ClassPathResource("train.csv");
+        final Resource file = new ClassPathResource("data.csv");
         final RedditClassifier redditClassifier = new RedditClassifier();
         redditClassifier.trainClassifier(file.getFile().getAbsolutePath());
         return redditClassifier;

spring-security-oauth/src/main/java/org/baeldung/reddit/classifier/RedditClassifier.java

@@ -26,12 +26,12 @@ public class RedditClassifier {
     public static int GOOD = 0;
     public static int BAD = 1;
     public static int MIN_SCORE = 10;
-    public static int NUM_OF_FEATURES = 1000;
 
     private final AdaptiveLogisticRegression classifier;
     private final FeatureVectorEncoder titleEncoder;
     private final FeatureVectorEncoder domainEncoder;
     private CrossFoldLearner learner;
+    private final int noOfFeatures;
     private double accuracy;
 
     private final int[] trainCount = { 0, 0 };
@@ -41,31 +41,44 @@ public class RedditClassifier {
     private final int[] correctCount = { 0, 0 };
 
     public RedditClassifier() {
-        classifier = new AdaptiveLogisticRegression(2, NUM_OF_FEATURES, new L2());
+        noOfFeatures = 1000;
+        classifier = new AdaptiveLogisticRegression(2, 1000, new L2());
         classifier.setPoolSize(150);
         titleEncoder = new AdaptiveWordValueEncoder("title");
         titleEncoder.setProbes(2);
         domainEncoder = new StaticWordValueEncoder("domain");
         domainEncoder.setProbes(1);
     }
 
+    public RedditClassifier(int poolSize, int noOfFeatures) {
+        this.noOfFeatures = noOfFeatures;
+        classifier = new AdaptiveLogisticRegression(2, noOfFeatures, new L2());
+        classifier.setPoolSize(poolSize);
+        titleEncoder = new AdaptiveWordValueEncoder("title");
+        titleEncoder.setProbes(1);
+        domainEncoder = new StaticWordValueEncoder("domain");
+        domainEncoder.setProbes(1);
+    }
+
     public void trainClassifier(String fileName) throws IOException {
         final List<NamedVector> vectors = extractVectors(readDataFile(fileName));
-        final int noOfTraining = (int) (RedditDataCollector.DATA_SIZE * 0.8);
+        final int size = vectors.size();
+        final int noOfTraining = (int) (size * 0.8);
         final List<NamedVector> trainingData = vectors.subList(0, noOfTraining);
-        final List<NamedVector> testData = vectors.subList(noOfTraining, RedditDataCollector.DATA_SIZE);
+        final List<NamedVector> testData = vectors.subList(noOfTraining, size);
         int category;
         for (final NamedVector vector : trainingData) {
             category = (vector.getName() == "GOOD") ? GOOD : BAD;
             classifier.train(category, vector);
             trainCount[category]++;
         }
         System.out.println("Training count ========= Good = " + trainCount[0] + " ___ Bad = " + trainCount[1]);
+        System.out.println("----------------------------------------------------------------- \n");
         evaluateClassifier(testData);
     }
 
     public Vector convertPost(String title, String domain, int hour) {
-        final Vector vector = new RandomAccessSparseVector(NUM_OF_FEATURES);
+        final Vector vector = new RandomAccessSparseVector(noOfFeatures);
         final List<String> words = Splitter.onPattern("\\W").omitEmptyStrings().splitToList(title);
         vector.set(0, hour);
         vector.set(1, words.size());
@@ -105,10 +118,10 @@ private void evaluateClassifier(List<NamedVector> vectors) throws IOException {
                 wrong++;
             }
         }
-        System.out.println("Eval count ========= Good = " + evalCount[0] + " ___ Bad = " + evalCount[1]);
-        System.out.println("Test result ======== Correct prediction = " + correct + " -----  Wrong prediction = " + wrong);
-        System.out.println("Test result ======== Correct Good = " + correctCount[0] + " -----  Correct Bad = " + correctCount[1]);
-        System.out.println("Test result ======== Good accuracy = " + (correctCount[0] / (evalCount[0] + 0.0)) + " ----- Bad accuracy = " + (correctCount[1] / (evalCount[1] + 0.0)));
+        System.out.println("Eval count   =================== Good = " + evalCount[0] + " ----- Bad = " + evalCount[1] + "\n");
+        System.out.println("Overall Evaluation ============= Correct prediction = " + correct + " -----  Wrong prediction = " + wrong);
+        System.out.println("Correctly Evaluated  =========== Correct Good = " + correctCount[0] + " -----  Correct Bad = " + correctCount[1]);
+        System.out.println("Correctly Evaluated (%) ======== Good accuracy = " + (correctCount[0] / (evalCount[0] + 0.0)) + " ----- Bad accuracy = " + (correctCount[1] / (evalCount[1] + 0.0)));
         this.accuracy = correct / (wrong + correct + 0.0);
     }
 
@@ -133,7 +146,7 @@ private List<NamedVector> extractVectors(List<String> lines) {
     private NamedVector extractVector(String line) {
         final String[] items = line.split(",");
         final String category = extractCategory(Integer.parseInt(items[0]));
-        final NamedVector vector = new NamedVector(new RandomAccessSparseVector(NUM_OF_FEATURES), category);
+        final NamedVector vector = new NamedVector(new RandomAccessSparseVector(noOfFeatures), category);
         final Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
         cal.setTimeInMillis(Long.parseLong(items[1]) * 1000);
 

spring-security-oauth/src/main/java/org/baeldung/reddit/classifier/RedditDataCollector.java

@@ -16,8 +16,8 @@
 import com.google.common.base.Splitter;
 
 public class RedditDataCollector {
-    public static final String TRAINING_FILE = "src/main/resources/train.csv";
-    public static final int DATA_SIZE = 8000;
+    public static final String DATA_FILE = "src/main/resources/data.csv";
+    public static final int DATA_SIZE = 20000;
     public static final int LIMIT = 100;
     public static final Long YEAR = 31536000L;
     private final Logger logger = LoggerFactory.getLogger(getClass());
@@ -45,10 +45,11 @@ public RedditDataCollector(String subreddit) {
     public void collectData() throws IOException {
         final int noOfRounds = DATA_SIZE / LIMIT;
         timestamp = System.currentTimeMillis() / 1000;
-        final FileWriter writer = new FileWriter(TRAINING_FILE);
+        final FileWriter writer = new FileWriter(DATA_FILE);
         writer.write("Score, Timestamp in utc, Number of wrods in title, Title, Domain \n");
         for (int i = 0; i < noOfRounds; i++) {
             getPosts(writer);
+            System.out.println(i);
         }
         writer.close();
     }