b danmu

Author: StephenMeng

src/main/java/com/stephen/lab/controller/crawler/BilibiliController.java

@@ -0,0 +1,83 @@
+package com.stephen.lab.controller.crawler;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.stephen.lab.constant.semantic.ResultEnum;
+import com.stephen.lab.model.others.BDanmu;
+import com.stephen.lab.model.others.Bilibili;
+import com.stephen.lab.service.crawler.BDanmuService;
+import com.stephen.lab.service.crawler.BilibiliService;
+import com.stephen.lab.util.HttpUtils;
+import com.stephen.lab.util.LogRecod;
+import com.stephen.lab.util.Response;
+import org.apache.commons.io.IOUtils;
+import org.apache.xerces.impl.xpath.regex.Match;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import sun.rmi.runtime.Log;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@RestController
+@RequestMapping("crawl_b")
+public class BilibiliController {
+    @Autowired
+    private BilibiliService bilibiliService;
+    @Autowired
+    private BDanmuService bDanmuService;
+
+    @RequestMapping("bilibili")
+    public Response parseVedioId() {
+        String url = "https://search.bilibili.com/api/search?search_type=all&keyword=%E5%95%A6%E5%95%A6%E5%95%A6";
+        try {
+            String jsonStr = HttpUtils.okrHttpGet(url);
+            JSONArray jsonArray = JSON.parseObject(jsonStr).getJSONObject("result").getJSONArray("video");
+            List<Bilibili> bilibiliList = jsonArray.toJavaList(Bilibili.class);
+            bilibiliList.forEach(bilibili -> {
+                bilibiliService.insert(bilibili);
+            });
+            return Response.success(bilibiliList);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return Response.error(ResultEnum.FAIL_PARAM_WRONG);
+    }
+
+    @RequestMapping("danmu")
+    public Response danmu() {
+        List<Bilibili> bilibiliList = bilibiliService.selectAll();
+        for (Bilibili bilibili : bilibiliList) {
+            crawlOneVedioDanmu(bilibili.getCid());
+        }
+        return Response.error(ResultEnum.FAIL_PARAM_WRONG);
+    }
+
+    private Response crawlOneVedioDanmu(Long cid) {
+        String url = "http://comment.bilibili.com/" + cid + ".xml";
+        try {
+            String xml = IOUtils.toString(HttpUtils.httpGet(url).getEntity().getContent());
+            Pattern danmuPattern = Pattern.compile("<d(.*?)d>");
+            Matcher danmuMatcher = danmuPattern.matcher(xml);
+            while  (danmuMatcher.find()) {
+                String content = danmuMatcher.group();
+                BDanmu danmu = new BDanmu();
+                danmu.setAid(cid);
+                danmu.setContent(content);
+                try {
+                    bDanmuService.insert(danmu);
+                } catch (Exception e) {
+                    LogRecod.print(cid);
+                    LogRecod.print(content);
+                }
+                LogRecod.print(content);
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+}

src/main/java/com/stephen/lab/controller/others/XieTianXuanController.java

@@ -1324,7 +1324,7 @@ public Response perplexity(int totalNum) {
         for (String content : contents) {
             N += content.split("#").length;
         }
-        for (int i = 1; i < totalNum; i++) {
+        for (int i = 6; i < totalNum; i++) {
             int topicNum = i * 10;
             trainAndSaveLdaModel(topicNum, kivaSimpleList);
             double p = LdaModel.getPerplexity(N, topicNum, thetaFile, twordsFile);

src/main/java/com/stephen/lab/controller/paper/OptimitalKivaController.java

@@ -0,0 +1,12 @@
+package com.stephen.lab.dao.crawler;
+
+import com.stephen.lab.model.others.BDanmu;
+import com.stephen.lab.util.BaseDao;
+import org.apache.ibatis.annotations.Mapper;
+
+/**
+ * Created by stephen on 2018/4/20.
+ */
+@Mapper
+public interface BDanmuDao extends BaseDao<BDanmu> {
+}

src/main/java/com/stephen/lab/dao/crawler/BDanmuDao.java

@@ -1,4 +1,4 @@
-package com.stephen.lab.dao.others;
+package com.stephen.lab.dao.crawler;
 
 import com.stephen.lab.model.others.Bilibili;
 import com.stephen.lab.util.BaseDao;

src/main/java/com/stephen/lab/dao/others/BilibiliDao.java renamed to src/main/java/com/stephen/lab/dao/crawler/BilibiliDao.java

@@ -0,0 +1,33 @@
+package com.stephen.lab.model.others;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Table;
+
+/**
+ * Created by stephen on 2018/4/20.
+ */
+@Entity
+@Table(name = "crawl_bilibili_danmu")
+public class BDanmu {
+    @Column(name = "aid")
+    private Long aid;
+    @Column(name = "content")
+    private String content;
+
+    public Long getAid() {
+        return aid;
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public void setAid(Long aid) {
+        this.aid = aid;
+    }
+
+    public void setContent(String content) {
+        this.content = content;
+    }
+}

src/main/java/com/stephen/lab/model/others/BDanmu.java

@@ -1,33 +1,63 @@
 package com.stephen.lab.model.others;
 
-import org.apache.xpath.operations.Bool;
-
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
 import java.util.Date;
 
+@Entity
+@Table(name = "crawl_bilibili")
 public class Bilibili {
+    @Id
+    @Column(name = "aid")
     private Long aid;
+    @Column(name = "arcran")
     private String arcran;
+    @Column(name = "arcurl")
     private String arcurl;
+    @Column(name = "author")
     private String author;
+    @Column(name = "badgepay")
     private Boolean badgepay;
+    @Column(name = "description")
     private String description;
+    @Column(name = "duration")
     private String duration;
+    @Column(name = "favorites")
     private Integer favorites;
+    @Column(name = "hit_columns")
     private String hit_columns;
+    @Column(name = "id")
     private Long id;
+    @Column(name = "mid")
     private Long mid;
+    @Column(name = "pic")
     private String pic;
+    @Column(name = "play")
     private Long play;
+    @Column(name = "pubdate")
     private Date pubdate;
+    @Column(name = "rank_score")
     private Long rank_score;
+    @Column(name = "review")
     private Integer review;
+    @Column(name = "senddate")
     private Date senddate;
+    @Column(name = "tag")
     private String tag;
+    @Column(name = "title")
     private String title;
+    @Column(name = "type")
     private String type;
+    @Column(name = "typeid")
     private String typeid;
+    @Column(name = "typename")
     private String typename;
+    @Column(name = "video_review")
     private Integer video_review;
+    @Column(name = "cid")
+    private Long cid;
 
     public Long getAid() {
         return aid;
@@ -212,4 +242,12 @@ public Integer getVideo_review() {
     public void setVideo_review(Integer video_review) {
         this.video_review = video_review;
     }
+
+    public Long getCid() {
+        return cid;
+    }
+
+    public void setCid(Long cid) {
+        this.cid = cid;
+    }
 }

src/main/java/com/stephen/lab/model/others/Bilibili.java

@@ -0,0 +1,7 @@
+package com.stephen.lab.service.crawler;
+
+import com.stephen.lab.model.others.BDanmu;
+
+public interface BDanmuService {
+    int insert(BDanmu bDanmu);
+}

src/main/java/com/stephen/lab/service/crawler/BDanmuService.java

@@ -0,0 +1,13 @@
+package com.stephen.lab.service.crawler;
+
+import com.stephen.lab.model.others.Bilibili;
+
+import java.util.List;
+
+public interface BilibiliService {
+    int insert(Bilibili bilibili);
+
+    List<Bilibili> selectAll();
+
+    int updateSelective(Bilibili b);
+}

src/main/java/com/stephen/lab/service/crawler/BilibiliService.java

@@ -0,0 +1,22 @@
+package com.stephen.lab.service.crawler.impl;
+
+import com.stephen.lab.dao.crawler.BDanmuDao;
+import com.stephen.lab.dao.crawler.BilibiliDao;
+import com.stephen.lab.model.others.BDanmu;
+import com.stephen.lab.model.others.Bilibili;
+import com.stephen.lab.service.crawler.BDanmuService;
+import com.stephen.lab.service.crawler.BilibiliService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class BDanmuServiceImpl implements BDanmuService {
+    @Autowired
+    private BDanmuDao danmuDao;
+
+
+    @Override
+    public int insert(BDanmu bDanmu) {
+        return danmuDao.insert(bDanmu);
+    }
+}

src/main/java/com/stephen/lab/service/crawler/impl/BDanmuServiceImpl.java

@@ -0,0 +1,30 @@
+package com.stephen.lab.service.crawler.impl;
+
+import com.stephen.lab.dao.crawler.BilibiliDao;
+import com.stephen.lab.model.others.Bilibili;
+import com.stephen.lab.service.crawler.BilibiliService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+@Service
+public class BilibiliServiceImpl implements BilibiliService {
+    @Autowired
+    private BilibiliDao bilibiliDao;
+
+    @Override
+    public int insert(Bilibili bilibili) {
+        return bilibiliDao.insert(bilibili);
+    }
+
+    @Override
+    public List<Bilibili> selectAll() {
+        return bilibiliDao.selectAll();
+    }
+
+    @Override
+    public int updateSelective(Bilibili b) {
+        return bilibiliDao.updateByPrimaryKeySelective(b);
+    }
+}