Merge pull request eugenp#195 from Doha2012/master

fix data collector

Author: Eugen

spring-security-oauth/pom.xml

@@ -185,7 +185,7 @@
             <artifactId>guava</artifactId>
             <version>${guava.version}</version>
         </dependency>
-
+        
         <!-- logging -->
 
         <dependency>

spring-security-oauth/src/main/java/org/baeldung/config/ServletInitializer.java

@@ -30,11 +30,9 @@ protected WebApplicationContext createRootApplicationContext() {
     @Override
     public void onStartup(ServletContext servletContext) throws ServletException {
         super.onStartup(servletContext);
-
         servletContext.addListener(new SessionListener());
         registerProxyFilter(servletContext, "oauth2ClientContextFilter");
         registerProxyFilter(servletContext, "springSecurityFilterChain");
-
     }
 
     private void registerProxyFilter(ServletContext servletContext, String name) {

spring-security-oauth/src/main/java/org/baeldung/config/SessionListener.java

@@ -3,6 +3,7 @@
 import javax.servlet.http.HttpSessionEvent;
 import javax.servlet.http.HttpSessionListener;
 
+import org.baeldung.reddit.util.MyFeatures;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -13,6 +14,7 @@ public class SessionListener implements HttpSessionListener {
     public void sessionCreated(HttpSessionEvent event) {
         logger.info("==== Session is created ====");
         event.getSession().setMaxInactiveInterval(30 * 60);
+        event.getSession().setAttribute("PREDICTION_FEATURE", MyFeatures.PREDICTION_FEATURE);
     }
 
     @Override

spring-security-oauth/src/main/java/org/baeldung/reddit/classifier/RedditClassifier.java

@@ -3,6 +3,8 @@
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.Calendar;
+import java.util.TimeZone;
 
 import org.apache.mahout.classifier.sgd.L2;
 import org.apache.mahout.classifier.sgd.OnlineLogisticRegression;
@@ -18,6 +20,7 @@ public class RedditClassifier {
 
     public static int GOOD = 0;
     public static int BAD = 1;
+    public static int MIN_SCORE = 5;
     private final OnlineLogisticRegression classifier;
     private final FeatureVectorEncoder titleEncoder;
     private final FeatureVectorEncoder domainEncoder;
@@ -44,7 +47,7 @@ public void trainClassifier(String fileName) throws IOException {
         }
 
         while ((line != null) && (line != "")) {
-            category = (line.startsWith("good")) ? GOOD : BAD;
+            category = extractCategory(line);
             trainCount[category]++;
             features = convertLineToVector(line);
             classifier.train(category, features);
@@ -76,7 +79,7 @@ public double evaluateClassifier() throws IOException {
         Vector features;
         String line = reader.readLine();
         while ((line != null) && (line != "")) {
-            category = (line.startsWith("good")) ? GOOD : BAD;
+            category = extractCategory(line);
             evalCount[category]++;
             features = convertLineToVector(line);
             result = classify(features);
@@ -94,12 +97,21 @@ public double evaluateClassifier() throws IOException {
     }
 
     // ==== private
+    private int extractCategory(String line) {
+        final int score = Integer.parseInt(line.substring(0, line.indexOf(';')));
+        return (score < MIN_SCORE) ? BAD : GOOD;
+    }
+
     private Vector convertLineToVector(String line) {
         final Vector features = new RandomAccessSparseVector(4);
         final String[] items = line.split(";");
+        final Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"));
+        cal.setTimeInMillis(Long.parseLong(items[1]) * 1000);
+        final int hour = cal.get(Calendar.HOUR_OF_DAY);
+
         titleEncoder.addToVector(items[3], features);
         domainEncoder.addToVector(items[4], features);
-        features.set(2, Integer.parseInt(items[1])); // hour of day
+        features.set(2, hour); // hour of day
         features.set(3, Integer.parseInt(items[2])); // number of words in the title
         return features;
     }

spring-security-oauth/src/main/java/org/baeldung/reddit/classifier/RedditDataCollector.java

@@ -2,9 +2,7 @@
 
 import java.io.FileWriter;
 import java.io.IOException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 
 import org.baeldung.reddit.util.UserAgentInterceptor;
@@ -20,57 +18,52 @@
 public class RedditDataCollector {
     public static final String TRAINING_FILE = "src/main/resources/train.csv";
     public static final String TEST_FILE = "src/main/resources/test.csv";
+    public static final int LIMIT = 100;
+    public static final Long YEAR = 31536000L;
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    private String postAfter;
+    private Long timestamp;
     private final RestTemplate restTemplate;
     private final String subreddit;
-    private final int minScore;
 
     public RedditDataCollector() {
         restTemplate = new RestTemplate();
         final List<ClientHttpRequestInterceptor> list = new ArrayList<ClientHttpRequestInterceptor>();
         list.add(new UserAgentInterceptor());
         restTemplate.setInterceptors(list);
-        subreddit = "all";
-        minScore = 4;
+        subreddit = "java";
     }
 
-    public RedditDataCollector(String subreddit, int minScore) {
+    public RedditDataCollector(String subreddit) {
         restTemplate = new RestTemplate();
         final List<ClientHttpRequestInterceptor> list = new ArrayList<ClientHttpRequestInterceptor>();
         list.add(new UserAgentInterceptor());
         restTemplate.setInterceptors(list);
         this.subreddit = subreddit;
-        this.minScore = minScore;
     }
 
     public void collectData() {
-        final int limit = 100;
         final int noOfRounds = 80;
+        timestamp = System.currentTimeMillis() / 1000;
         try {
             final FileWriter writer = new FileWriter(TRAINING_FILE);
             for (int i = 0; i < noOfRounds; i++) {
-                getPosts(limit, writer);
+                getPosts(writer);
             }
             writer.close();
 
             final FileWriter testWriter = new FileWriter(TEST_FILE);
-            getPosts(limit, testWriter);
+            getPosts(testWriter);
             testWriter.close();
         } catch (final Exception e) {
             logger.error("write to file error", e);
         }
     }
 
-    // ==== private
-
-    private void getPosts(int limit, FileWriter writer) {
-        String fullUrl = "http://www.reddit.com/r/" + subreddit + "/new.json?limit=" + limit;
-        if (postAfter != null) {
-            fullUrl += "&count=" + limit + "&after=" + postAfter;
-        }
+    // ==== Private
 
+    private void getPosts(FileWriter writer) {
+        final String fullUrl = "http://www.reddit.com/r/" + subreddit + "/search.json?sort=new&q=timestamp:" + (timestamp - YEAR) + ".." + timestamp + "&restrict_sr=on&syntax=cloudsearch&limit=" + LIMIT;
         try {
             final JsonNode node = restTemplate.getForObject(fullUrl, JsonNode.class);
             parseNode(node, writer);
@@ -82,22 +75,18 @@ private void getPosts(int limit, FileWriter writer) {
     }
 
     private void parseNode(JsonNode node, FileWriter writer) throws IOException {
-        postAfter = node.get("data").get("after").asText();
-        System.out.println(postAfter);
         String line;
-        String category;
         List<String> words;
-        final SimpleDateFormat df = new SimpleDateFormat("HH");
+        int score;
         for (final JsonNode child : node.get("data").get("children")) {
-            category = (child.get("data").get("score").asInt() < minScore) ? "bad" : "good";
+            score = child.get("data").get("score").asInt();
             words = Splitter.onPattern("\\W").omitEmptyStrings().splitToList(child.get("data").get("title").asText());
-            final Date date = new Date(child.get("data").get("created_utc").asLong() * 1000);
+            timestamp = child.get("data").get("created_utc").asLong();
 
-            line = category + ";";
-            line += df.format(date) + ";";
+            line = score + ";";
+            line += timestamp + ";";
             line += words.size() + ";" + Joiner.on(' ').join(words) + ";";
             line += child.get("data").get("domain").asText() + "\n";
-
             writer.write(line);
         }
     }

spring-security-oauth/src/main/java/org/baeldung/reddit/util/MyFeatures.java

@@ -0,0 +1,17 @@
+package org.baeldung.reddit.util;
+
+public enum MyFeatures {
+
+    PREDICTION_FEATURE(false);
+
+    private boolean active;
+
+    private MyFeatures(boolean active) {
+        this.active = active;
+    }
+
+    public boolean isActive() {
+        return this.active;
+    }
+
+}