有戏的filter

Author: StephenMeng

src/main/java/com/stephen/lab/controller/paper/KivaController.java

@@ -59,8 +59,8 @@ public class KivaController {
     private String trainFilePath = "C:\\Users\\stephen\\Desktop\\svm\\train\\train-%s.txt";
     private String modelFilePath = "C:\\Users\\stephen\\Desktop\\svm\\model\\model-%s.txt";
     private String testFilePath = "C:\\Users\\stephen\\Desktop\\svm\\test\\testdata.txt";
-    private String predictFilePath = "C:\\Users\\stephen\\Desktop\\svm\\predict\\predict-%s.txt";
     private static final int TAG_FREQ = 1;
+    private int F_VALUE_NUM = 6;
 
     @ApiOperation(value = "解析loans", notes = "将数据解析到数据库")
     @RequestMapping(value = "init", method = RequestMethod.GET)
@@ -203,7 +203,7 @@ public Response genTagByKnn(@RequestParam("num") int num) throws IOException {
             Collections.sort(kivaResult.getTokenList(), Comparator.comparing(Token::getWeight));
             List<KivaResult> selectResult = getKnnSimilarDoc(kivaResult, kivaResults, num);
             List<Token> sortedTags = getSortedTags(selectResult);
-            sortedTags=sublist(sortedTags,num);
+            sortedTags = sublist(sortedTags, num);
             updateDatabaseOfGenTag(kivaResult.getKiva().getId(), sortedTags, TagType.KNN);
         });
         LogRecod.print("knn finished");
@@ -448,7 +448,11 @@ public Response tranformToSimple(int num) {
                 simple.setOriginalDescription(des);
                 simple.setStandardDescription(Joiner.on("#").join(ct));
                 simple.setTags(normalizeTags(kiva.getTags()));
-                kivaService.insertKivaSimple(simple);
+                try {
+                    kivaService.insertKivaSimple(simple);
+                } catch (Exception e) {
+                    kivaService.updateSimpleSelective(simple);
+                }
             }
         });
         return Response.success("");
@@ -760,7 +764,7 @@ private double pValue(List<KivaSimple> simpleList, int tagType) throws IOExcepti
             GenTag genTag = JSONObject.parseObject(kivaSimple.getGenTags(), GenTag.class);
             List<String> originTags = splitString(kivaSimple.getTags());
             List<Token> originalTokens = Lists.transform(originTags, Token::new);
-            List<Token> genTags = getTagsByTagType(tagType, genTag);
+            List<Token> genTags = sublist(getTagsByTagType(tagType, genTag), F_VALUE_NUM);
             precise += same(genTags, originalTokens);
             all += genTags.size();
         }
@@ -788,7 +792,7 @@ private double rValue(List<KivaSimple> simpleList, int tagType) throws IOExcepti
             GenTag genTag = JSONObject.parseObject(kivaSimple.getGenTags(), GenTag.class);
             List<String> originTags = splitString(kivaSimple.getTags());
             List<Token> originalTokens = Lists.transform(originTags, Token::new);
-            List<Token> genTags = getTagsByTagType(tagType, genTag);
+            List<Token> genTags = sublist(getTagsByTagType(tagType, genTag), F_VALUE_NUM);
             recall += same(genTags, originalTokens);
             all += originTags.size();
         }
@@ -1129,7 +1133,7 @@ public Response genTagByLDA(int num) {
             KivaResult kivaResult = new KivaResult(kivaSimple);
             List<KivaResult> selectResult = getLDASimilarDoc(kivaResult, kivaList, score, docThetas, num);
             List<Token> sortedTags = getSortedTags(selectResult);
-            sortedTags=sublist(sortedTags,num);
+            sortedTags = sublist(sortedTags, num);
             updateDatabaseOfGenTag(kivaResult.getKiva().getId(), sortedTags, TagType.LDA);
         }
         return Response.success(docThetas);
@@ -1210,9 +1214,11 @@ private List<Token> getSimlarDocTokens(String text, List<Token> toCompare, List<
             List<Long> validIds = simplesContainsTag(token.getWord());
             List<KivaResult> krs = filterValid(kivaResults, validIds);
             double score = computeToSimlerResult(krs, toCompare);
-            Token t = new Token(token.getWord());
-            t.setWeight(score);
-            result.add(t);
+            if(score>0.005) {
+//                Token t = new Token(token.getWord());
+//                t.setWeight(score);
+                result.add(token);
+            }
         }
         Collections.sort(result, Comparator.comparing(Token::getWeight).reversed());
 

src/main/java/com/stephen/lab/util/nlp/NLPIRUtil.java

@@ -62,9 +62,9 @@ public static List<String> cutwords(String docContent) {
 //                String ne = token.get(CoreAnnotations.NamedEntityTagAnnotation.class);    // 获取命名实体识别结果
                 // 获取词形还原结果
                 String lemma = token.get(CoreAnnotations.LemmaAnnotation.class);
-                if (!pos.startsWith("V")) {
+//                if (!pos.startsWith("V")) {
                     result.add(lemma);
-                }
+//                }
             }
         }
         return result;