写了个samples

Author: zhangyingwei

.gitignore

@@ -17,5 +17,5 @@
 /.idea
 /target
 cockroach.iml
-*.imi
 
+*.iml

cockroach-annotation/cockroach-annotation.iml

@@ -10,7 +10,7 @@
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="module" module-name="cockroach-core" />
+    <orderEntry type="library" name="Maven: com.github.zhangyingwei:cockroach-core:1.0.5.03-Beta" level="project" />
     <orderEntry type="library" name="Maven: com.squareup.okhttp3:okhttp:3.8.1" level="project" />
     <orderEntry type="library" name="Maven: com.squareup.okio:okio:1.13.0" level="project" />
     <orderEntry type="library" name="Maven: net.sf.json-lib:json-lib:jdk15:2.4" level="project" />

cockroach-annotation/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>cockroach</artifactId>
         <groupId>com.github.zhangyingwei</groupId>
-        <version>1.0.5.03-Beta</version>
+        <version>1.0.5.04-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

cockroach-core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>cockroach</artifactId>
         <groupId>com.github.zhangyingwei</groupId>
-        <version>1.0.5.03-Beta</version>
+        <version>1.0.5.04-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

cockroach-samples/cockroach-samples.iml

@@ -11,7 +11,7 @@
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="module" module-name="cockroach-core" />
+    <orderEntry type="library" name="Maven: com.github.zhangyingwei:cockroach-core:1.0.5.03-Beta" level="project" />
     <orderEntry type="library" name="Maven: com.squareup.okhttp3:okhttp:3.8.1" level="project" />
     <orderEntry type="library" name="Maven: com.squareup.okio:okio:1.13.0" level="project" />
     <orderEntry type="library" name="Maven: net.sf.json-lib:json-lib:jdk15:2.4" level="project" />

cockroach-samples/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>cockroach</artifactId>
         <groupId>com.github.zhangyingwei</groupId>
-        <version>1.0.5.03-Beta</version>
+        <version>1.0.5.04-Beta</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

cockroach-samples/src/main/java/com/zhangyingwei/cockroach/samples/oschina/OschinaApplicatoin.java

@@ -0,0 +1,29 @@
+package com.zhangyingwei.cockroach.samples.oschina;
+
+import com.zhangyingwei.cockroach.CockroachApplication;
+import com.zhangyingwei.cockroach.annotation.AppName;
+import com.zhangyingwei.cockroach.annotation.EnableAutoConfiguration;
+import com.zhangyingwei.cockroach.annotation.Store;
+import com.zhangyingwei.cockroach.annotation.ThreadConfig;
+import com.zhangyingwei.cockroach.executer.task.Task;
+import com.zhangyingwei.cockroach.queue.CockroachQueue;
+import com.zhangyingwei.cockroach.queue.TaskQueue;
+import com.zhangyingwei.cockroach.samples.oschina.store.OsChinaStore;
+
+/**
+ * Created by zhangyw on 2018/2/26.
+ */
+
+@EnableAutoConfiguration
+@AppName("开源中国博客爬虫")
+@ThreadConfig(num = 10,sleep = 500)
+@Store(OsChinaStore.class)
+public class OschinaApplicatoin {
+    public static void main(String[] args) throws Exception {
+        CockroachQueue queue = TaskQueue.of();
+        for (int i = 1; i <= 10; i++) {
+            queue.push(new Task("https://www.oschina.net/action/ajax/get_more_recommend_blog?classification=0&p="+i,"oschina.blog").retry(5));
+        }
+        CockroachApplication.run(OschinaApplicatoin.class, queue);
+    }
+}

cockroach-samples/src/main/java/com/zhangyingwei/cockroach/samples/oschina/store/OsChinaStore.java

@@ -0,0 +1,32 @@
+package com.zhangyingwei.cockroach.samples.oschina.store;
+
+import com.zhangyingwei.cockroach.executer.response.TaskResponse;
+import com.zhangyingwei.cockroach.executer.task.Task;
+import com.zhangyingwei.cockroach.store.IStore;
+import org.jsoup.select.Elements;
+
+import java.util.List;
+
+/**
+ * Created by zhangyw on 2018/2/26.
+ */
+public class OsChinaStore implements IStore {
+    @Override
+    public void store(TaskResponse response) throws Exception {
+        if (response.isGroup("oschina.blog")) {
+            response.select(".item").forEach(item -> {
+                String href = item.select(".blog-title-link").attr("href");
+                try {
+                    response.getQueue().push(new Task(href,"oschina.blog.item"));
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            });
+        } else if (response.isGroup("oschina.blog.item")) {
+            Elements content = response.select(".blog-content");
+            String title = content.select(".title").text().replace("原 荐 ","");
+            String autor = content.select(".name").text();
+            System.out.println(String.format("文章标题: %s  作者: %s",title,autor));
+        }
+    }
+}

cockroach-samples/src/main/resources/log4j.properties

@@ -0,0 +1,7 @@
+# Root logger option
+log4j.rootLogger=INFO,stdout
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}] %-5p %c{1}:%L - %m%n

cockroach-test/cockroach-test.iml

@@ -10,7 +10,7 @@
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="module" module-name="cockroach-core" />
+    <orderEntry type="library" name="Maven: com.github.zhangyingwei:cockroach-core:1.0.5.03-Beta" level="project" />
     <orderEntry type="library" name="Maven: com.squareup.okhttp3:okhttp:3.8.1" level="project" />
     <orderEntry type="library" name="Maven: com.squareup.okio:okio:1.13.0" level="project" />
     <orderEntry type="library" name="Maven: net.sf.json-lib:json-lib:jdk15:2.4" level="project" />