diff --git a/crawler/.DS_Store b/crawler/.DS_Store
deleted file mode 100644
index 58af3a86a..000000000
Binary files a/crawler/.DS_Store and /dev/null differ
diff --git a/crawler/.idea/compiler.xml b/crawler/.idea/compiler.xml
deleted file mode 100644
index 61a9130cd..000000000
--- a/crawler/.idea/compiler.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/.idea/crawler.iml b/crawler/.idea/crawler.iml
deleted file mode 100644
index d6ebd4805..000000000
--- a/crawler/.idea/crawler.iml
+++ /dev/null
@@ -1,9 +0,0 @@
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/.idea/misc.xml b/crawler/.idea/misc.xml
deleted file mode 100644
index 348936d64..000000000
--- a/crawler/.idea/misc.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/.idea/modules.xml b/crawler/.idea/modules.xml
deleted file mode 100644
index 58377ce74..000000000
--- a/crawler/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/.idea/workspace.xml b/crawler/.idea/workspace.xml
deleted file mode 100644
index 321eed596..000000000
--- a/crawler/.idea/workspace.xml
+++ /dev/null
@@ -1,251 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
- DEFINITION_ORDER
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1503302070409
-
-
- 1503302070409
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/.classpath b/crawler/dailyScheduledCrawler/fetch_networks/.classpath
deleted file mode 100644
index 22e721e39..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/.classpath
+++ /dev/null
@@ -1,36 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/.project b/crawler/dailyScheduledCrawler/fetch_networks/.project
deleted file mode 100644
index 43f566a3c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/.project
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
- fetchnetworks
-
-
-
-
-
- org.eclipse.jdt.core.javabuilder
-
-
-
-
- org.maven.ide.eclipse.maven2Builder
-
-
-
-
- org.eclipse.m2e.core.maven2Builder
-
-
-
-
-
- org.maven.ide.eclipse.maven2Nature
- org.eclipse.jdt.core.javanature
- org.eclipse.m2e.core.maven2Nature
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs b/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 839d647ee..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding/=UTF-8
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs b/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs
deleted file mode 100644
index 5a0ad22d2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs
+++ /dev/null
@@ -1,2 +0,0 @@
-eclipse.preferences.version=1
-line.separator=\n
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs b/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 308d3cd9a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,14 +0,0 @@
-#Sun Jan 04 15:44:05 CST 2015
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.7
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs b/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs
deleted file mode 100644
index f897a7f1c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-activeProfiles=
-eclipse.preferences.version=1
-resolveWorkspaceProjects=true
-version=1
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/51cto_blog.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/51cto_blog.sh
deleted file mode 100644
index a2e234083..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/51cto_blog.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='51cto_blog'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/apache.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/apache.sh
deleted file mode 100644
index c62bdb5cc..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/apache.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='apache'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/bytes.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/bytes.sh
deleted file mode 100644
index 55a2f39bc..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/bytes.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='bytes'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_news.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_news.sh
deleted file mode 100644
index 573fd7151..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_news.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='cnblogs_news'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_solved.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_solved.sh
deleted file mode 100644
index f7c9d03f6..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_solved.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='cnblogs_q_solved'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh
deleted file mode 100644
index 82267a995..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='cnblogs_q_unsolved'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/code_project.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/code_project.sh
deleted file mode 100644
index fa217100e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/code_project.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='code_project'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_ask.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_ask.sh
deleted file mode 100644
index 8017f556c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_ask.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='csdn_ask'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_blog.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_blog.sh
deleted file mode 100644
index a4d9f1099..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_blog.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='csdn_blog'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_topic.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_topic.sh
deleted file mode 100644
index 2a07e1a8f..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_topic.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='csdn_topic'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx256m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/dewen_question.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/dewen_question.sh
deleted file mode 100644
index 56edd8338..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/dewen_question.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='dewen_question'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/freecode.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/freecode.sh
deleted file mode 100644
index 2b0feeb44..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/freecode.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='freecode'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/gna.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/gna.sh
deleted file mode 100644
index 56f5db521..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/gna.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='gna'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_post.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_post.sh
deleted file mode 100644
index 955bbf074..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_post.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='ibm_post'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_project.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_project.sh
deleted file mode 100644
index 872de0841..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_project.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='ibm_project'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_ask.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_ask.sh
deleted file mode 100644
index df0dadbf4..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_ask.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='iteye_ask'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_blog.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_blog.sh
deleted file mode 100644
index f2c485b2f..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_blog.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='iteye_blog'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/lagou.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/lagou.sh
deleted file mode 100644
index 3ce108072..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/lagou.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='lagou'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/linuxtone.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/linuxtone.sh
deleted file mode 100644
index 10642a713..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/linuxtone.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='linuxtone'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/lupaworld.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/lupaworld.sh
deleted file mode 100644
index 0f978b3f9..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/lupaworld.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='lupaworld'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/neitui.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/neitui.sh
deleted file mode 100644
index e7781806e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/neitui.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='neitui'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/openhub.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/openhub.sh
deleted file mode 100644
index 128f0fa2e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/openhub.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='openhub'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_project.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_project.sh
deleted file mode 100644
index 4e85901c3..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_project.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='oschina_project'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_question.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_question.sh
deleted file mode 100644
index 1df9a7ec9..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_question.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='oschina_question'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/phpchina.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/phpchina.sh
deleted file mode 100644
index a75679ada..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/phpchina.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='phpchina'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/log4j.xml b/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/log4j.xml
deleted file mode 100644
index fb9cbc5a6..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/log4j.xml
+++ /dev/null
@@ -1,79 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml b/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml
deleted file mode 100644
index ada70d40f..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 21600000
-
-
-
- 21600000
-
-
-
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext.xml b/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext.xml
deleted file mode 100644
index 7d83874af..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/testDBlogger.xml b/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/testDBlogger.xml
deleted file mode 100644
index bc2391d0b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/resources/testDBlogger.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/sample_start.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/sample_start.sh
deleted file mode 100644
index 1037be1d2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/sample_start.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='freecode'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/slashdot.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/slashdot.sh
deleted file mode 100644
index 132e8ca10..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/slashdot.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='slashdot'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/softpedia.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/softpedia.sh
deleted file mode 100644
index cf8de56f4..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/softpedia.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='softpedia'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/sourceforge.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/sourceforge.sh
deleted file mode 100644
index b96002bd9..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/sourceforge.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='sourceforge'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/stackoverflow.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/stackoverflow.sh
deleted file mode 100644
index 1a40da282..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/stackoverflow.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='stackoverflow'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn48m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/bin/start_all.sh b/crawler/dailyScheduledCrawler/fetch_networks/bin/start_all.sh
deleted file mode 100644
index e901cc634..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/bin/start_all.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-sh bin/cnblogs_news.sh
-sh bin/cnblogs_q_solved.sh
-sh bin/cnblogs_q_unsolved.sh
-sh bin/csdn_ask.sh
-sh bin/csdn_blog.sh
-sh bin/csdn_topic.sh
-sh bin/dewen_question.sh
-sh bin/iteye_ask.sh
-sh bin/openhub.sh
-sh bin/oschina_project.sh
-sh bin/oschina_question.sh
-sh bin/sourceforge.sh
-sh bin/stackoverflow.sh
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-core-fix.jar b/crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-core-fix.jar
deleted file mode 100644
index 39d45c42e..000000000
Binary files a/crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-core-fix.jar and /dev/null differ
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-extension-fix.jar b/crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-extension-fix.jar
deleted file mode 100644
index 7a204b496..000000000
Binary files a/crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-extension-fix.jar and /dev/null differ
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/ossean-bak.xml b/crawler/dailyScheduledCrawler/fetch_networks/ossean-bak.xml
deleted file mode 100644
index cac15888e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/ossean-bak.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-
-Spider
-1
-one
-5
-1
-true
-20000
-5
-30000
-5000
-5
-Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-10000
-http://www.oschina.net/question/
-http://www\.oschina\.net/question\?catalog=[12]\&show=active\&p=\d+
-http://www\.oschina\.net/question/\d+_\d+
-0
-10
-10000
-html_test
-target_url
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/pom.xml b/crawler/dailyScheduledCrawler/fetch_networks/pom.xml
deleted file mode 100644
index cc181748b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/pom.xml
+++ /dev/null
@@ -1,202 +0,0 @@
-
-
-
- 4.0.0
-
- net.trustie
- fetchnetworks
- 1.0-SNAPSHOT
-
-
- UTF-8
-
- /
- 3.1.1.RELEASE
- 3.1.0.RELEASE
-
-
-
- ${basedir}/src/main/java
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
- 1.7
- 1.7
-
-
-
- maven-assembly-plugin
- 2.5.1
-
-
- src/main/assembly/assembly.xml
-
-
-
-
-
-
-
-
- xerces
- xerces
- 2.4.0
-
-
- dom4j
- dom4j
- 1.6.1
-
-
- org.json
- json
- 20141113
-
-
-
- com.google.guava
- guava
- 14.0
-
-
-
- us.codecraft
- xsoup
- 0.2.4
-
-
-
- com.github.dreamhead
- moco-core
- 0.10.0
-
-
-
- org.slf4j
- slf4j-log4j12
- 1.7.7
-
-
-
-
-
- commons-collections
- commons-collections
- 3.2.1
-
-
-
- org.assertj
- assertj-core
- 1.7.0
-
-
-
- org.apache.commons
- commons-io
- 1.3.2
-
-
-
- com.jayway.jsonpath
- json-path
- 0.8.1
-
-
-
- com.alibaba
- fastjson
- 1.2.3
-
-
-
- org.jsoup
- jsoup
- 1.7.2
-
-
- org.apache.httpcomponents
- httpclient
- 4.3.4
-
-
- org.springframework
- spring-jdbc
- ${spring-version}
-
-
- org.apache.commons
- commons-lang3
- 3.1
-
-
- javax.servlet
- servlet-api
- 2.5
-
-
- mysql
- mysql-connector-java
- 5.1.18
-
-
- commons-dbcp
- commons-dbcp
- 1.3
-
-
- junit
- junit
- 4.7
- test
-
-
-
- org.mybatis
- mybatis
- 3.1.1
-
-
-
- org.mybatis
- mybatis-spring
- 1.1.1
-
-
-
- org.springframework
- spring-test
- ${spring-version}
- test
-
-
-
- net.trustie
- webmagic-core-fix
- 1.0
- system
- ${project.basedir}/lib/webmagic-core-fix.jar
-
-
-
- net.trustie
- webmagic-extension-fix
- 1.0
- system
- ${project.basedir}/lib/webmagic-extension-fix.jar
-
-
-
- org.seleniumhq.selenium
- selenium-java
- 2.42.2
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/51cto_blog.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/51cto_blog.xml
deleted file mode 100644
index d165b25b8..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/51cto_blog.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- 51cto
- 1
- 30
- div.rightbox>div.r_li>h4>a
- http://blog.51cto.com
- http://blog.51cto.com/original/0/
-
- 16
- 86400
- 0
-
-
-
-
- 51cto_blog
- 20000
- 30000
- 600000
- 1200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/Copy of sample_site.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/Copy of sample_site.xml
deleted file mode 100644
index 77da3d5e0..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/Copy of sample_site.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-
-
-
- Spider
-
- csdn_topic
-
- 1
-
- 10
-
- div.content>div.list_1>ul>li>a
-
- http://bbs.csdn.net
-
- http://bbs.csdn.net/tech_hot_topics?page=
-
-
- 0
-
- 50
-
- 20000
-
- 10000
- 20000
-
- 1800000
- 7200000
-
- 3600000
- 7200000
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/apache.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/apache.xml
deleted file mode 100644
index cc7ba5c14..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/apache.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
-
-
-
-
- 1
- 1
- table>tbody>tr>td.body>div>div>ul>li> a
- http://projects.apache.org
- http://projects.apache.org/indexes/alpha.html
-
- 1
- 864000
- 0
-
-
-
-
- apache
- 10000
- 30000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml
deleted file mode 100644
index 42216357b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- 51cto_blog
-
- 1
-
- 1
-
- true
-
- 30
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://blog.51cto.com/original/0/1
-
-
-
- div.rightbox>div.r_li>h4>a
-
- 0
-
- 10
-
- 10000
-
- http://blog.51cto.com
-
- http://blog.51cto.com/original/0/
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 16
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of apache.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of apache.xml
deleted file mode 100644
index 8553844a8..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of apache.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- apache
-
- 1
-
- 1
-
- true
-
- 30
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://projects.apache.org/indexes/alpha.html
-
-
-
- table>tbody>tr>td.body>div>div>ul>li> a
-
- 0
-
- 10
-
- 10000
-
- http://projects.apache.org/indexes/alpha.html
-
- http://projects.apache.org/indexes/alpha.html
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 1
-
-
- 8640000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml
deleted file mode 100644
index 1b1e2c389..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- cnblogs_news
-
- 1
-
- 1
-
- true
-
- 4845
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://news.cnblogs.com/n/page/1
-
-
-
- div#news_list>div.news_block>div.content>h2.news_entry>a
-
- 0
-
- 10
-
- 10000
-
- http://news.cnblogs.com
-
- http://news.cnblogs.com/n/page/
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 5
-
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml
deleted file mode 100644
index 9319eb0ae..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- cnblogs_q_solved
-
- 1
-
- 1
-
- true
-
- 160
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 4000
-
- http://q.cnblogs.com/list/solved?page=1
-
-
-
- div.one_entity>div.news_item>h2.news_entry>a
-
- 0
-
- 10
-
- 10000
-
- http://q.cnblogs.com
-
- http://q.cnblogs.com/list/solved?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml
deleted file mode 100644
index 72768f9b8..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-
-
-
-
- Spider
-
- 1
-
- cnblogs_q_unsolved
-
- 1
-
- 1
-
- true
-
- 160
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 4000
-
- http://q.cnblogs.com/list/unsolved?page=1
-
-
-
- div.one_entity>div.news_item>h2.news_entry>a
-
- 0
-
- 10
-
- 10000
-
- http://q.cnblogs.com
-
- http://q.cnblogs.com/list/unsolved?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of code_project.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of code_project.xml
deleted file mode 100644
index 3bb9f53a9..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of code_project.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- codeproject
-
- 1
-
- 1
-
- true
-
- 21
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=1
-
-
-
- div.content-list-item>div.title>a
-
- 0
-
- 10
-
- 10000
-
- http://www.codeproject.com
-
- http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 1
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml
deleted file mode 100644
index 4eb3d847f..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- dewen_question
-
- 1
-
- 1
-
- true
-
- 1114
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.dewen.io/questions?page=1
-
-
-
- div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a
-
- 0
-
- 10
-
- 10000
-
- http://www.dewen.io
-
- http://www.dewen.io/questions?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 1
-
-
-
- 432000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of gna.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of gna.xml
deleted file mode 100644
index f1bb2cef0..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of gna.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- gna
-
- 1
-
- 1
-
- true
-
- 1348
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=1&max_rows=1#results
-
-
-
- table>tbody>tr>td>a
-
- 0
-
- 10
-
- 10000
-
- http://gna.org
-
- http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=
-
- &max_rows=1#results
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 200
-
-
- 432000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml
deleted file mode 100644
index 0687a8e34..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- lupaworld
-
- 1
-
- 2
-
- true
-
- 479
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.lupaworld.com/forum-13746-2.html
-
-
-
- table#threadlisttableid>tbody>tr>th>a
-
- 0
-
- 10
-
- 10000
-
- http://www.lupaworld.com
-
- http://www.lupaworld.com/forum-13746-
-
- .html
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 2592000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of openhub.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of openhub.xml
deleted file mode 100644
index f48e5dc87..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of openhub.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- openhub
-
- 1
-
- 1
-
- true
-
- 66784
-
- 5
-
- 60000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- https://www.openhub.net/p?page=1&q=&sort=users
-
-
-
- div.project>h2.title>a
-
- 0
-
- 10
-
- 60000
-
- https://www.openhub.net
-
- https://www.openhub.net/p?page=
-
- &q=&sort=users
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 500
-
- 100000
-
-
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of phpchina.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of phpchina.xml
deleted file mode 100644
index 712aa22d2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of phpchina.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- phpchina
-
- 1
-
- 1
-
- true
-
- 1289
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://bbs.phpchina.com/forum-17-1.html
-
-
-
- table#threadlisttableid>tbody>tr>th>a.s.xst
-
- 0
-
- 10
-
- 10000
-
- http://bbs.phpchina.com
-
- http://bbs.phpchina.com/forum-17-
-
- .html
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 1296000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of softpedia.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of softpedia.xml
deleted file mode 100644
index 9d9c0eb87..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of softpedia.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- softpedia
-
- 1
-
- 1
-
- true
-
- 30
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://linux.softpedia.com/index1.shtml
-
-
-
- div#sjmp>div>h4>a
-
- 0
-
- 10
-
- 10000
-
- http://linux.softpedia.com
-
- http://linux.softpedia.com/index
-
- .shtml
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 10
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml
deleted file mode 100644
index ba738497c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- sourceforge
-
- 1
-
- 1
-
- true
-
- 17621
-
- 5
-
- 60000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 10000
-
- http://sourceforge.net/directory/?page=1
-
-
-
- ul.projects>li>div.project_info>header>a
-
- 0
-
- 10
-
- 10000
-
- http://sourceforge.net
-
- http://sourceforge.net/directory/?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 500
-
- 250000
-
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml
deleted file mode 100644
index 21713db99..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- stackoverflow
-
- 1
-
- 1
-
- true
-
- 25000
-
- 5
-
- 60000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 10000
-
- http://stackoverflow.com/questions?pagesize=50&sort=newest&page=1
-
-
-
- div#mainbar>div#questions>div.question-summary>div.summary>h3>a
-
- 0
-
- 10
-
- 10000
-
- http://stackoverflow.com
-
- http://stackoverflow.com/questions?pagesize=50&sort=newest&page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 250
-
- 20000
-
- 2000
- 3000
-
-
- 1800000
- 2400000
-
-
- 7200000
- 14400000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml
deleted file mode 100644
index 0f13b714a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- linuxtone
-
- 1
-
- 1
-
- true
-
- 64
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://bbs.linuxtone.org/forum-15-1.html
-
-
-
- table>tbody>tr>th>a
-
- 0
-
- 10
-
- 10000
-
- http://bbs.linuxtone.org
-
- http://bbs.linuxtone.org/forum-15-
-
- .html
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 2592000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml
deleted file mode 100644
index 8c9e1d8ad..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- iteye_blog
-
- 1
-
- 1
-
- true
-
- 15
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.iteye.com/blogs?page=1
-
-
-
- div.content>h3>a
-
- 0
-
- 10
-
- 10000
-
- http://www.iteye.com
-
- http://www.iteye.com/blogs?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/bytes.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/bytes.xml
deleted file mode 100644
index 45f49d0eb..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/bytes.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- question
- 1
- 2000
- table#threadslist>tbody>tr>td>div>div.threadbit_title>a
- http://bytes.com
- http://bytes.com/answers/
- /
- 2
- 86400
- 0
-
-
-
-
- bytes
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_news.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_news.xml
deleted file mode 100644
index 16565d865..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_news.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-
-
-
-
-
-
- 1
- 4845
- div#news_list>div.news_block>div.content>h2.news_entry>a
- http://news.cnblogs.com
- http://news.cnblogs.com/n/page/
-
- 5
- 86400
- 0
-
-
-
-
- cnblogs_news
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_solved.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_solved.xml
deleted file mode 100644
index 17598cd7c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_solved.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
-
-
-
- 1
- 160
- div.one_entity>div.news_item>h2.news_entry>a
- http://q.cnblogs.com
- http://q.cnblogs.com/list/solved?page=
-
- 2
- 86400
- 0
-
-
-
-
- cnblogs_q_solved
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml
deleted file mode 100644
index 1676a68a0..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 30
- div.one_entity>div.news_item>h2.news_entry>a
- http://q.cnblogs.com
- http://q.cnblogs.com/list/unsolved?page=
-
- 2
- 86400
- 0
-
-
-
-
- cnblogs_q_unsolved
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/code_project.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/code_project.xml
deleted file mode 100644
index 580834ed4..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/code_project.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- 1
- 259
- div.content-list-item>div.title>a
- http://www.codeproject.com
-
- http://www.codeproject.com/script/Articles/Latest.aspx?la_as=366&la_app=20&la_minscore=1.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=
-
- 1
- 86400
- 0
-
-
-
-
- codeproject
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_ask.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_ask.xml
deleted file mode 100644
index f45d7a50e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_ask.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 2496
- div.common_con>div.questions_detail_con>dl>dt>a
- http://ask.csdn.net
- http://ask.csdn.net/?page=
-
- 25
- 86400
- 0
-
-
-
-
- csdn_ask
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_blog.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_blog.xml
deleted file mode 100644
index 63c25bb29..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_blog.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 101
- div.main_center>div.blog_list>h1>a:not(.category)
- http://blog.csdn.net
- http://blog.csdn.net/index.html?&page=
-
- 3
- 86400
- 0
-
-
-
-
- csdn_blog
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_topic.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_topic.xml
deleted file mode 100644
index 1749acd3d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_topic.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-
-
-
-
-
- mobile
- 1
- 808
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/Mobile?page=
-
- 8
- 86400
- 0
-
-
-
- CloudComputing
- 1
- 22
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/CloudComputing?page=
-
- 1
- 86400
- 0
-
-
-
- java
- 1
- 2555
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/Java?page=
-
- 4
- 86400
- 0
-
-
-
- web
- 1
- 1853
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/WebDevelop?page=
-
- 3
- 86400
- 0
-
-
-
- OtherLanguage
- 1
- 336
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/OtherLanguage?page=
-
- 1
- 86400
- 0
-
-
-
-
- csdn_topic
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/dewen_question.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/dewen_question.xml
deleted file mode 100644
index 6038c0293..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/dewen_question.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
-
-
-
-
- 1
- 1114
- div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a
- http://www.dewen.io
- http://www.dewen.io/questions?page=
-
- 1
- 432000
- 0
-
-
-
-
- dewen_question
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/freecode.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/freecode.xml
deleted file mode 100644
index 943e6ded2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/freecode.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
- Spider
-
- 5
-
- freecode
-
- 5
-
- 1
-
- true
-
- 4799
-
- 5
-
- 30000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 10000
-
- http://www.freecode.com/projects?page=1
-
-
-
- div.project>h2.release-head>a
-
- 0
-
- 10
-
- 60000
-
- http://www.freecode.com
-
- http://www.freecode.com/projects?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 5
-
- 20000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/gna.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/gna.xml
deleted file mode 100644
index 712713f46..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/gna.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 1348
- table>tbody>tr>td>a
- http://gna.org
- http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=
- &max_rows=1#results
- 100
- 86400
- 0
-
-
-
-
- gna
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_post.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_post.xml
deleted file mode 100644
index 4d55bc4fd..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_post.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- blog
- 1
- 138
- table>tbody>tr>td>a
- http://www.ibm.com
- http://www.ibm.com/developerworks/cn/views/opensource/libraryview.jsp?site_id=10&contentarea_by=%E6%89%80%E6%9C%89%E4%B8%93%E5%8C%BA&sort_by=&sort_order=2&start=
- 00&end=13874&topic_by=-1&product_by=&type_by=%E6%89%80%E6%9C%89%E7%B1%BB%E5%88%AB&show_abstract=false&search_by=&industry_by=&series_title_by=
- 1
- 6048000
- 0
-
-
-
-
- ibm_post
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_project.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_project.xml
deleted file mode 100644
index 3e0e2be41..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_project.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 1
- table>tbody>tr>td>a
- http://www.ibm.com
- http://www.ibm.com/developerworks/views/opensource/projects.jsp
-
- 1
- 864000
- 0
-
-
-
-
- ibm_project
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_ask.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_ask.xml
deleted file mode 100644
index 72778075a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_ask.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
- Spider
-
- 5
-
- iteye_ask
-
- 5
-
- 1
-
- true
-
- 5329
-
- 5
-
- 30000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.iteye.com/ask?page=1
-
-
-
- div#ask_list>div.question-summary>div.summary>h3>a
-
- 0
-
- 10
-
- 60000
-
- http://www.iteye.com/
-
- http://www.iteye.com/ask?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 50
-
- 20000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_bbs.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_bbs.xml
deleted file mode 100644
index 4dfddb45d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_bbs.xml
+++ /dev/null
@@ -1,78 +0,0 @@
-
-
-
-
- mobile
- z
- 111
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/mobile?page=
-
- 16
- 1296000
- 0
-
-
-
- language
- 1
- 242
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/language?page=
-
- 1
- 1296000
- 0
-
-
-
- web
- 1
- 320
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/web?page=
-
- 1
- 1296000
- 0
-
-
- java
- 1
- 754
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/Java?page=
-
- 1
- 1296000
- 0
-
-
- tech
- 1
- 242
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/Tech?page=
-
- 1
- 1296000
- 0
-
-
-
-
- iteye_bbs
- 30000
- 50000
- 3600000
- 7200000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_blog.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_blog.xml
deleted file mode 100644
index b45fa450e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_blog.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
- blog
- 1
- 15
- div.content>h3>a
- http://www.iteye.com
- http://www.iteye.com/blogs?page=
-
- 2
- 86400
- 0
-
-
-
-
- iteye_blog
- 30000
- 50000
- 3600000
- 7200000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/javaforge.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/javaforge.xml
deleted file mode 100644
index 0bdc05cf9..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/javaforge.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- javaforge
-
- 1
-
- 1
-
- true
-
- 31
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://javaforge.com/listProjects.spr?page=1&displayAllProjects=on
-
-
-
- table#entry>tbody>tr>td:nth-child(1) > a
-
- 0
-
- 10
-
- 10000
-
- http://javaforge.com
-
- http://javaforge.com/listProjects.spr?page=
-
- &displayAllProjects=on
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 10
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/lagou.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/lagou.xml
deleted file mode 100644
index 4bfade221..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/lagou.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- houduan
- 1
- 30
- #container > div.content>ul> li> div> div > a
- http://www.lagou.com
- http://www.lagou.com/jobs/list_%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91?kd=%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91&spc=1&pl=&gj=&xl=&yx=&gx=&st=&labelWords=label&lc=&workAddress=&city=%E5%85%A8%E5%9B%BD&requestId=&pn=
-
- 16
- 86400
- 0
-
-
-
-
- lagou
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/linuxtone.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/linuxtone.xml
deleted file mode 100644
index 1930f0fa4..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/linuxtone.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-
-
-
-
-
- System
- 1
- 64
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-15-
- .html
- 1
- 2592000
- 0
-
-
-
- server
- 1
- 30
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-22-
- .html
- 1
- 2592000
- 0
-
-
-
- dbserver
- 1
- 32
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-24-
- .html
- 1
- 2592000
- 0
-
-
-
- yngz
- 1
- 32
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-49-
- .html
- 1
- 2592000
- 0
-
-
-
- fzjh
- 1
- 24
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-26-
- .html
- 1
- 2592000
- 0
-
-
-
-
- linuxtone
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/lupaworld.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/lupaworld.xml
deleted file mode 100644
index 57d3e0bc2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/lupaworld.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
-
-
- 1
- 479
- table#threadlisttableid>tbody>tr>th>a
- http://www.lupaworld.com
- http://www.lupaworld.com/forum-13746-
- .html
- 2
- 2592000
- 0
-
-
-
-
- lupaworld
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/neitui.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/neitui.xml
deleted file mode 100644
index 7854326cb..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/neitui.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 610
- ul>li>div.cont>div.jobnote.clearfix > div.jobnote-l>a
- http://www.neitui.me
- http://www.neitui.me/neitui/type=all&page=
- .html
- 16
- 86400
- 0
-
-
-
-
- neitui
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/open_open.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/open_open.xml
deleted file mode 100644
index 400a46b3f..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/open_open.xml
+++ /dev/null
@@ -1,152 +0,0 @@
-
-
-
-
-
- dev
- 1
- 370
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/1?pn=
-
- 1
- 86400
- 0
-
-
-
-
- language
- 1
- 652
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/36?pn=
-
- 1
- 86400
- 0
-
-
-
-
- website
- 1
- 41
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/57?pn=
-
- 1
- 86400
- 0
-
-
-
-
- server
- 1
- 145
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/75?pn=
-
- 1
- 86400
- 0
-
-
-
- database
- 1
- 102
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/90?pn=
-
- 1
- 86400
- 0
-
-
-
- plugin
- 1
- 53
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/52?pn=
-
- 1
- 86400
- 0
-
-
-
-
- OS
- 1
- 42
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/155?pn=
-
- 1
- 86400
- 0
-
-
-
-
-
- manage
- 1
- 12
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/319?pn=
-
- 1
- 86400
- 0
-
-
-
- dev_manage
- 1
- 9
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/159?pn=
-
- 1
- 86400
- 0
-
-
-
- util
- 1
- 7
- div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a
- http://www.open-open.com
- http://www.open-open.com/lib/list/322?pn=
-
- 1
- 86400
- 0
-
-
-
-
- open_open
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/openhub.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/openhub.xml
deleted file mode 100644
index 6d0d9fe80..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/openhub.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-
-
-
-
-
-
- 1
- 66784
- div.well>h2.title>a
- https://www.openhub.net
-
-
- https://www.openhub.net/p?page=
- &q=&sort=new
- 8
- 86400
-
- 0
-
-
-
-
- openhub
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_project.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_project.xml
deleted file mode 100644
index 6585b52e5..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_project.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 2044
- div.ProjectList>ul.List>li>h3>a
- http://www.oschina.net
- http://www.oschina.net/project/list?prefix=&sort=time&p=
-
- 1
- 86400
- 0
-
-
-
-
- oschina_project
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_question.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_question.xml
deleted file mode 100644
index d059d37c4..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_question.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
- 1
- 2361
- ul.list>li.question>div.question-detail>strong>a
- http://www.oschina.net
- http://www.oschina.net/question?catalog=1&show=active&p=
-
- 1
- 86400
- 0
-
-
-
-
- oschina_question
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/ossean.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/ossean.xml
deleted file mode 100644
index f35470b83..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/ossean.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
- Spider
-
- 3
-
- ossean
-
- 3
-
- 1
-
- true
-
- 5777
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 500
-
- http://localhost/open_source_projects?page=1
-
-
-
- ul.projects>li.project-table>div.root>a
-
- 0
-
- 10
-
- 10000
-
- http://localhost
-
- http://localhost/open_source_projects?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 5
-
- 0
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/phpchina.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/phpchina.xml
deleted file mode 100644
index ef48a9514..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/phpchina.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
-
-
-
-
- ask
- 1
- 1289
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-17-
- .html
- 2
- 1296000
- 0
-
-
-
- server_os
- 1
- 91
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-195-
- .html
- 2
- 1296000
- 0
-
-
-
- web
- 1
- 81
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-213-
- .html
- 2
- 1296000
- 0
-
-
-
- db
- 1
- 43
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-196-
- .html
- 2
- 1296000
- 0
-
-
-
-
- phpchina
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/sample_site.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/sample_site.xml
deleted file mode 100644
index f8224e89d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/sample_site.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-
-
-
- Spider
-
- 3
-
- csdn_topic
-
- 3
-
- 1
-
- true
-
- 10
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 4000
-
- http://bbs.csdn.net/tech_hot_topics?page=1
-
-
-
- div.content>div.list_1>ul>li>a
-
- 0
-
- 10
-
- 10000
-
- http://bbs.csdn.net
-
- http://bbs.csdn.net/tech_hot_topics?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 50
-
- 20000
-
-
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/slashdot.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/slashdot.xml
deleted file mode 100644
index 330938e6a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/slashdot.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-
-
-
-
- 1
- 66
-
- #firehoselist>article>header>h2>span>a
- http://slashdot.org
- http://slashdot.org/?page=
-
- 2
- 86400
- 0
-
-
-
-
- slashdot
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/softpedia.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/softpedia.xml
deleted file mode 100644
index 5ace82f8c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/softpedia.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-
-
-
-
-
- linux
- 1
- 30
- h4>a
- http://linux.softpedia.com
- http://linux.softpedia.com/index
- .shtml
- 3
- 86400
- 0
-
-
-
- win
- 1
- 30
- h4>a
- http://win.softpedia.com
- http://win.softpedia.com/index
- .shtml
- 3
- 86400
- 0
-
-
-
- mac
- 1
- 35
- h4>a
- http://mac.softpedia.com
- http://mac.softpedia.com/index
- .shtml
- 5
- 86400
- 0
-
-
-
- windows-phone
- 1
- 46
- h4>a
- http://mobile.softpedia.com
- http://mobile.softpedia.com/windows-phone,
-
- 5
- 86400
- 0
-
-
-
- android
- 1
- 35
- h4>a
- http://mobile.softpedia.com
- http://mobile.softpedia.com/android,
-
- 1
- 86400
- 0
-
-
-
- ios
- 1
- 22
- h4>a
- http://mobile.softpedia.com
- http://mobile.softpedia.com/ios,
-
- 1
- 86400
- 0
-
-
-
- driver
- 1
- 30
- h4>a
- http://drivers.softpedia.com
- http://drivers.softpedia.com/index
- .shtml
- 10
- 86400
- 0
-
-
-
- game
- 1
- 30
- h4>a
- http://games.softpedia.com
- http://games.softpedia.com/index
- .shtml
- 2
- 86400
- 0
-
-
-
- webscripts
- 1
- 30
- h4>a
- http://webscripts.softpedia.com
- http://webscripts.softpedia.com/index/latest-scripts-3
- -0-0.html
- 3
- 86400
- 0
-
-
-
-
-
- softpedia
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/sourceforge.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/sourceforge.xml
deleted file mode 100644
index b71b11503..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/sourceforge.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-
-
-
-
-
-
- 1
- 17621
- ul.projects>li>div.project_info>header>a
- http://sourceforge.net
-
-
-
- http://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update&page=
-
-
-
- 8
- 86400
- 0
-
-
-
-
- sourceforge
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/spider_test.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/spider_test.xml
deleted file mode 100644
index 7a880b14b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/spider_test.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- database
- 1
- 20
- div#news_list>div.news_block>div.content>h2.news_entry>a
- http://localhost:8080/SpiderTest
- http://localhost:8080/SpiderTest/database?page=
-
- 15
- 300
- 0
-
-
- os
- 1
- 20
- div#news_list>div.news_block>div.content>h2.news_entry>a
- http://localhost:8080/SpiderTest
- http://localhost:8080/SpiderTest/os?page=
-
- 15
- 300
- 0
-
-
-
-
- SpiderTes
- 1000
- 3000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sites/stackoverflow.xml b/crawler/dailyScheduledCrawler/fetch_networks/sites/stackoverflow.xml
deleted file mode 100644
index 7aa902c32..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sites/stackoverflow.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 25000
- div#mainbar>div#questions>div.question-summary>div.summary>h3>a
- http://stackoverflow.com
- http://stackoverflow.com/questions?pagesize=50&sort=newest&page=
-
- 250
- 86400
- 0
-
-
-
-
- stackoverflow
- 2000
- 3000
- 900000
- 1800000
- 7200000
- 14400000
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/spider.conf.xml b/crawler/dailyScheduledCrawler/fetch_networks/spider.conf.xml
deleted file mode 100644
index 4c4103350..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/spider.conf.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-
-
-
-Spider
-1
-ques_oschina
-2
-1
-true
-2
-5
-10000
-5000
-5
-Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36
-10000
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sql/detail_html.sql b/crawler/dailyScheduledCrawler/fetch_networks/sql/detail_html.sql
deleted file mode 100644
index 412202765..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sql/detail_html.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
-`crawledTime` datetime NULL DEFAULT NULL ,
-`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=1
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sql/error_page.sql b/crawler/dailyScheduledCrawler/fetch_networks/sql/error_page.sql
deleted file mode 100644
index 98585115e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sql/error_page.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
-`crawledTime` datetime NULL DEFAULT NULL ,
-`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=8388
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sql/list_html.sql b/crawler/dailyScheduledCrawler/fetch_networks/sql/list_html.sql
deleted file mode 100644
index 1ea3a0622..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sql/list_html.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
-`crawledTime` datetime NULL DEFAULT NULL ,
-`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`extracted` bigint(20) NULL DEFAULT NULL ,
-PRIMARY KEY (`id`),
-INDEX `pagemd5` (`pageMd5`) USING BTREE
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=25
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sql/pointers.sql b/crawler/dailyScheduledCrawler/fetch_networks/sql/pointers.sql
deleted file mode 100644
index e8185e406..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sql/pointers.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`table_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`pointer` int(11) NULL DEFAULT NULL ,
-`created_at` datetime NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=3
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/sql/url.sql b/crawler/dailyScheduledCrawler/fetch_networks/sql/url.sql
deleted file mode 100644
index 626ba4b67..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/sql/url.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`timestamp` bigint(20) NULL DEFAULT NULL ,
-`extractedTime` datetime NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=1
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/assembly/assembly.xml b/crawler/dailyScheduledCrawler/fetch_networks/src/main/assembly/assembly.xml
deleted file mode 100644
index baf8ba912..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/assembly/assembly.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
- jar-with-dependencies-without-resources
-
- dir
-
- false
-
-
- /
- false
- false
- runtime
-
-
- /
- false
- false
- system
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/META-INF/MANIFEST.MF b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/META-INF/MANIFEST.MF
deleted file mode 100644
index c34e50601..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/META-INF/MANIFEST.MF
+++ /dev/null
@@ -1,3 +0,0 @@
-Manifest-Version: 1.0
-Main-Class: net.trustie.webmagic.crawler.OSChinaQuestionCrawler
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/CSSPathExtractor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/CSSPathExtractor.java
deleted file mode 100644
index f7201cb3a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/CSSPathExtractor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.selector.Html;
-
-public class CSSPathExtractor implements Extractor{
- private String path = "";
-
- /**
- * @param path
- */
- public CSSPathExtractor(String path) {
- this.path = path;
- }
-
- @Override
- public Set extract(Html html) {
- // TODO Auto-generated method stub
- Document doc = html.getDocument();
- Elements eles = doc.select(path);
- Set urlSet = new HashSet();
- for(Element e :eles){
- //针对gnaurl以两个点开头
- String url = e.attr("href");
- if(url.startsWith("http://gna.org/..")){
- int pos = url.indexOf("..");
- url = url.substring(0,pos) + url.substring(pos+3,url.length());
- }
-
- urlSet.add(url);
- }
-
- return urlSet;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/Extractor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/Extractor.java
deleted file mode 100644
index 242448ae4..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/Extractor.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.Set;
-
-import us.codecraft.webmagic.selector.Html;
-
-public interface Extractor {
- public Set extract(Html html);
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/URLPatternExtractor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/URLPatternExtractor.java
deleted file mode 100644
index 24908cdbf..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/URLPatternExtractor.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import net.trustie.webmagic.utils.StringUtils;
-import us.codecraft.webmagic.selector.Html;
-
-public class URLPatternExtractor implements Extractor{
- private String pattern = "";
-
-
- /**
- * @param pattern
- */
- public URLPatternExtractor(String pattern) {
- this.pattern = pattern;
- }
-
-
- @Override
- public Set extract(Html html) {
- // TODO Auto-generated method stub
- List postUrls = html.links().regex(pattern).all();
-
- return StringUtils.removeDuplicate(postUrls);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/XPathExtractor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/XPathExtractor.java
deleted file mode 100644
index 7fb7a9f1c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/XPathExtractor.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.List;
-import java.util.Set;
-
-import net.trustie.webmagic.utils.StringUtils;
-import us.codecraft.webmagic.selector.Html;
-
-public class XPathExtractor implements Extractor {
- private String path = "";
-
- /**
- * @param path
- */
- public XPathExtractor(String path) {
- this.path = path;
- }
-
- @Override
- public Set extract(Html html) {
- // TODO Auto-generated method stub
- List urlList = html.xpath(path).all();
- return StringUtils.removeDuplicate(urlList);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Configure.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Configure.java
deleted file mode 100644
index a3b71587e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Configure.java
+++ /dev/null
@@ -1,600 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.utils.SubSite;
-
-import org.apache.log4j.Logger;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.InvalidPropertiesFormatException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Queue;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-public class Configure {
- private int sleepTimeThread = 5000;
-
- //--长时间睡眠时候的睡眠时间的上下限 -->
- private int minlongSleepTime = 1800000;
- private int maxlongSleepTime = 7200000;
-
- //长时间睡眠时候的睡眠间隔的上下限 -->
- private int minlongSleepInterval = 3600000;
- private int maxlongSleepInterval = 7200000;
- //用以设置请求间隔的两个阈值以产生随机数 (毫秒)
- private int minInterval = 10000;
- private int maxInterval = 20000;
-
- private int retryTimes = 0;
- private int cycleRetryTimes = 0;
- private int timeOut = 5000;
- private String userAgent = "UserAgent.Browser";
- private String charset;
- private int threadNum = 5;
- private boolean isSpawnUrl = true;
- private int startPageIndex = 1;
- private int endInPageIndex = 6;
- private int pageF = 1;
- private int sleepTimeSpider = 36000;
- private String domain;
- private String startUrl;
- private String urlList;
- private String urlPost;
- private int offset;
- private int limitLine;
- private int noUrlWaitTime;
- private String fixAllRelativeHrefs;
- private Properties prop = new Properties();
- private String prefix;
- private String postfix;
- private String extractMethod;
- private String proxyIp;
- private String proxyPort;
- private int mode;
- private int incrementalPages;
- private int incrementSleepTime;
- //private List subSites = new LinkedList();
- private Queue subSites = new LinkedList();
- Logger log4j = Logger.getLogger(Configure.class);
- Document doc = null;
- public Configure() {
- try {
- FileInputStream fis = null;
- fis = new FileInputStream("spider.conf.xml");
- prop.loadFromXML(fis);
- prop.list(System.out);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
-
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- public static void main(String args[]){
- Configure conf = new Configure("spider_test");
- }
- public Configure(String confPath) {
- /*confPath = confPath + ".xml";
- try {
- URL url = ClassLoader.getSystemResource(confPath);
- InputStream is = url.openStream();
- prop.loadFromXML(is);
- // prop.list(System.out);
- } catch (FileNotFoundException e) {
- log4j.error("FileNotFoundException & loading default config xml! & no default config file");
- e.printStackTrace();
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();*/
- confPath = confPath + ".xml";
- URL url = ClassLoader.getSystemResource(confPath);
- InputStream is;
- try {
- is = url.openStream();
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder db = dbf.newDocumentBuilder();
- doc = db.parse(is);
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (ParserConfigurationException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (SAXException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- config();
- }
-
-
- private void config() {
-
- //star modify----------------------------
- Element root = doc.getDocumentElement();
- //获取站点
- NodeList sites = ((Element)root.getElementsByTagName("subSites").item(0)).getElementsByTagName("subSite");
- NodeList properties = ((Element)root.getElementsByTagName("properties").item(0)).getChildNodes();
- for(int i =0; i < sites.getLength(); i++){
- //System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
- Node subSite= sites.item(i);
- addSubSite(subSite,subSites);
- }
- //用map存储properties ,然后再赋值;
- Map propertiesMap = new HashMap();
- for(int i =0; i < properties.getLength(); i++){
- //System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
- propertiesMap.put(properties.item(i).getNodeName(), properties.item(i).getTextContent());
- }
- this.domain = propertiesMap.get("domain");
- this.minInterval = Integer.parseInt(propertiesMap.get("minInterval")) ;
- this.maxInterval = Integer.parseInt(propertiesMap.get("maxInterval")) ;
-
- this.minlongSleepTime = Integer.parseInt(propertiesMap.get("minlongSleepTime")) ;
- this.maxlongSleepTime = Integer.parseInt(propertiesMap.get("maxlongSleepTime")) ;
-
- this.minlongSleepInterval = Integer.parseInt(propertiesMap.get("minlongSleepInterval")) ;
- this.maxlongSleepInterval= Integer.parseInt(propertiesMap.get("maxlongSleepInterval")) ;
- //System.out.println(propertiesMap); this.retryTimes = 5;
- this.timeOut = 20000;
- this.sleepTimeThread = 0;
- this.cycleRetryTimes = 5;
- this.pageF = 1;
- this.threadNum = 1;
- this.isSpawnUrl = true;
- this.offset = 0;
- this.limitLine = 10;
- this.noUrlWaitTime = 10000;
- this.extractMethod = "CSSPath";
- this.proxyIp = "192.168.245.1";
- this.proxyPort = "3128";
- this.userAgent = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
-
- //目前url修正和信息页抽取共用一个模板
- this.urlPost = this.subSites.peek().getUrlPost();
- this.fixAllRelativeHrefs = this.subSites.peek().getFixAllRelativeHrefs();
- //end of star modify-------------------------
-
- /*this.sleepTimeThread = Integer.parseInt(prop
- .getProperty("sleepTimeThread"));
-
-
- this.timeOut = Integer.parseInt(prop.getProperty("timeOut"));
- this.retryTimes = Integer.parseInt(prop.getProperty("retryTimes"));
- this.cycleRetryTimes = Integer.parseInt(prop
- .getProperty("cycleRetryTimes"));
- this.threadNum = Integer.parseInt(prop.getProperty("threadNum"));
- this.startPageIndex = Integer.parseInt(prop
- .getProperty("startPageIndex"));
- this.endInPageIndex = Integer.parseInt(prop
- .getProperty("endInPageIndex"));
- this.pageF = Integer.parseInt(prop.getProperty("pageF"));
- this.sleepTimeSpider = Integer.parseInt(prop
- .getProperty("sleepTimeSpider"));
- this.isSpawnUrl = Boolean.parseBoolean(prop.getProperty("isSpawnUrl"));
- this.userAgent = prop.getProperty("userAgent");
- this.domain = prop.getProperty("domain");
- this.startUrl = prop.getProperty("startUrl");
- this.urlList = prop.getProperty("urlList");
- this.urlPost = prop.getProperty("urlPost");
- this.offset = Integer.parseInt(prop.getProperty("offset"));
- this.limitLine = Integer.parseInt(prop.getProperty("limitLine"));
- this.noUrlWaitTime = Integer
- .parseInt(prop.getProperty("noUrlWaitTime"));
-
- this.fixAllRelativeHrefs = prop.getProperty("fixAllRelativeHrefs");
- this.prefix = prop.getProperty("prefixUrl");
- this.postfix = prop.getProperty("postfixUrl");
- this.extractMethod = prop.getProperty("extractMethod");
- this.proxyIp = prop.getProperty("proxyIp");
- this.proxyPort =prop.getProperty("proxyPort");
- this.mode = Integer.parseInt(prop.getProperty("mode"));
- this.incrementalPages = Integer.parseInt(prop.getProperty("incrementalPages"));
- this.incrementSleepTime = Integer.parseInt(prop.getProperty("incrementSleepTime"));
-
- this.minInterval = Integer.parseInt(prop.getProperty("minInterval"));
- this.maxInterval = Integer.parseInt(prop.getProperty("maxInterval"));*/
- }
-
-
-
- private void addSubSite(Node site, Queue subSites2) {
- NodeList sitePros = site.getChildNodes();
- Map siteProMap = new HashMap();
- for(int i =0; i < sitePros.getLength(); i++){
- //System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
- siteProMap.put(sitePros.item(i).getNodeName(), sitePros.item(i).getTextContent());
- }
-
- //System.out.println(siteProMap);
- SubSite subSite = new SubSite();
-
- subSite.setName(siteProMap.get("name"));
- subSite.setStartPageIndex(Integer.parseInt(siteProMap.get("startPageIndex")));
- subSite.setEndInPageIndex(Integer.parseInt(siteProMap.get("endInPageIndex")));
- subSite.setFixAllRelativeHrefs(siteProMap.get("fixAllRelativeHrefs"));
- subSite.setPrefixUrl(siteProMap.get("prefixUrl"));
- subSite.setPostfixUrl(siteProMap.get("postfixUrl"));
- subSite.setUrlPost(siteProMap.get("urlPost"));
- subSite.setIncrementalPages(Integer.parseInt(siteProMap.get("incrementalPages")));
- subSite.setIncrementSleepTime(Integer.parseInt(siteProMap.get("incrementSleepTime")));
-
- subSites2.offer(subSite);
- //System.out.println(subSite.getStartPageIndex());
- }
-
- /**
- * @return the incrementSleepTime
- */
- public int getIncrementSleepTime() {
- return incrementSleepTime;
- }
-
- /**
- * @param incrementSleepTime the incrementSleepTime to set
- */
- public void setIncrementSleepTime(int incrementSleepTime) {
- this.incrementSleepTime = incrementSleepTime;
- }
-
- public int getSleepTimeThread() {
- return sleepTimeThread;
- }
-
- public void setSleepTimeThread(int sleepTimeThread) {
- this.sleepTimeThread = sleepTimeThread;
- }
-
- public int getRetryTimes() {
- return retryTimes;
- }
-
- public void setRetryTimes(int retryTimes) {
- this.retryTimes = retryTimes;
- }
-
- public int getCycleRetryTimes() {
- return cycleRetryTimes;
- }
-
- public void setCycleRetryTimes(int cycleRetryTimes) {
- this.cycleRetryTimes = cycleRetryTimes;
- }
-
- public int getTimeOut() {
- return timeOut;
- }
-
- public void setTimeOut(int timeOut) {
- this.timeOut = timeOut;
- }
-
- public String getUserAgent() {
- return userAgent;
- }
-
- public void setUserAgent(String userAgent) {
- this.userAgent = userAgent;
- }
-
- public String getCharset() {
- return charset;
- }
-
- public void setCharset(String charset) {
- this.charset = charset;
- }
-
- public int getThreadNum() {
- return threadNum;
- }
-
- public void setThreadNum(int threadNum) {
- this.threadNum = threadNum;
- }
-
- public boolean isSpawnUrl() {
- return isSpawnUrl;
- }
-
- public void setSpawnUrl(boolean isSpawnUrl) {
- this.isSpawnUrl = isSpawnUrl;
- }
-
- public int getStartPageIndex() {
- return startPageIndex;
- }
-
- public void setStartPageIndex(int startPageIndex) {
- this.startPageIndex = startPageIndex;
- }
-
- public int getEndInPageIndex() {
- return endInPageIndex;
- }
-
- public void setEndInPageIndex(int endInPageIndex) {
- this.endInPageIndex = endInPageIndex;
- }
-
- public int getPageF() {
- return pageF;
- }
-
- public void setPageF(int pageF) {
- this.pageF = pageF;
- }
-
- public int getSleepTimeSpider() {
- return sleepTimeSpider;
- }
-
- public void setSleepTimeSpider(int sleepTimeSpider) {
- this.sleepTimeSpider = sleepTimeSpider;
- }
-
- public String getDomain() {
- return domain;
- }
-
- public void setDomain(String domain) {
- this.domain = domain;
- }
-
- public String getStartUrl() {
- return startUrl;
- }
-
- public void setStartUrl(String startUrl) {
- this.startUrl = startUrl;
- }
-
- public String getUrlList() {
- return urlList;
- }
-
- public void setUrlList(String urlList) {
- this.urlList = urlList;
- }
-
- public String getUrlPost() {
- return urlPost;
- }
-
- public void setUrlPost(String urlPost) {
- this.urlPost = urlPost;
- }
-
- public Properties getProp() {
- return prop;
- }
-
- public void setProp(Properties prop) {
- this.prop = prop;
- }
-
- public Logger getLog4j() {
- return log4j;
- }
-
- public void setLog4j(Logger log4j) {
- this.log4j = log4j;
- }
-
- public int getOffset() {
- return offset;
- }
-
- public void setOffset(int offset) {
- this.offset = offset;
- }
-
- public int getLimitLine() {
- return limitLine;
- }
-
- public void setLimitLine(int limitLine) {
- this.limitLine = limitLine;
- }
-
- public int getNoUrlWaitTime() {
- return noUrlWaitTime;
- }
-
- public void setNoUrlWaitTime(int noUrlWaitTime) {
- this.noUrlWaitTime = noUrlWaitTime;
- }
-
-
-
-
- public String getFixAllRelativeHrefs() {
- return fixAllRelativeHrefs;
- }
-
- public void setFixAllRelativeHrefs(String fixAllRelativeHrefs) {
- this.fixAllRelativeHrefs = fixAllRelativeHrefs;
- }
-
- public String getPrefix() {
- return prefix;
- }
-
- public void setPrefix(String prefix) {
- this.prefix = prefix;
- }
-
- public String getPostfix() {
- return postfix;
- }
-
- public void setPostfix(String postfix) {
- this.postfix = postfix;
- }
-
- public String getExtractMethod() {
- return extractMethod;
- }
-
- public void setExtractMethod(String extraMethod) {
- this.extractMethod = extraMethod;
- }
-
-
- /**
- * @return the proxyIp
- */
- public String getProxyIp() {
- return proxyIp;
- }
-
- /**
- * @return the proxyPort
- */
- public String getProxyPort() {
- return proxyPort;
- }
-
- /**
- * @param proxyIp the proxyIp to set
- */
- public void setProxyIp(String proxyIp) {
- this.proxyIp = proxyIp;
- }
-
- /**
- * @param proxyPort the proxyPort to set
- */
- public void setProxyPort(String proxyPort) {
- this.proxyPort = proxyPort;
- }
-
- /**
- * @return the mode
- */
- public int getMode() {
- return mode;
- }
-
- /**
- * @param mode the mode to set
- */
- public void setMode(int mode) {
- this.mode = mode;
- }
-
- /**
- * @return the incrementalPages
- */
- public int getIncrementalPages() {
- return incrementalPages;
- }
-
- /**
- * @param incrementalPages the incrementalPages to set
- */
- public void setIncrementalPages(int incrementalPages) {
- this.incrementalPages = incrementalPages;
- }
-
-
-
- //每次请求的时间间隔
- public int getMinInterval() {
- return minInterval;
- }
-
- public void setMinInterval(int minInterval) {
- this.minInterval = minInterval;
- }
-
- public int getMaxInterval() {
- return maxInterval;
- }
-
- public void setMaxInterval(int maxInterval) {
- this.maxInterval = maxInterval;
- }
-
- public int getMinlongSleepTime() {
- return minlongSleepTime;
- }
-
- public void setMinlongSleepTime(int minlongSleepTime) {
- this.minlongSleepTime = minlongSleepTime;
- }
-
- public int getMaxlongSleepTime() {
- return maxlongSleepTime;
- }
-
- public void setMaxlongSleepTime(int maxlongSleepTime) {
- this.maxlongSleepTime = maxlongSleepTime;
- }
-
- public int getMinlongSleepInterval() {
- return minlongSleepInterval;
- }
-
- public void setMinlongSleepInterval(int minlongSleepInterval) {
- this.minlongSleepInterval = minlongSleepInterval;
- }
-
- public int getMaxlongSleepInterval() {
- return maxlongSleepInterval;
- }
-
- public void setMaxlongSleepInterval(int maxlongSleepInterval) {
- this.maxlongSleepInterval = maxlongSleepInterval;
- }
-
- public int getSubSiteNum() {
- return this.subSites.size();
- }
-
-
- // 获取下一个要处理的站点,并更新当前状态
- public SubSite nexSite() {
- SubSite subSite = this.subSites.poll();
- this.subSites.offer( subSite);
-
- this.startPageIndex = subSite.getStartPageIndex();
- this.endInPageIndex = subSite.getEndInPageIndex();
- this.fixAllRelativeHrefs = subSite.getFixAllRelativeHrefs();
- this.prefix = subSite.getPrefixUrl();
- this.postfix = subSite.getPostfixUrl();
- this.urlPost = subSite.getUrlPost();
- this.incrementalPages = subSite.getIncrementalPages();
- this.incrementSleepTime = subSite.getIncrementSleepTime();
-
-
- return subSite;
- }
-
-
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Crawler.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Crawler.java
deleted file mode 100644
index e22a49baf..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Crawler.java
+++ /dev/null
@@ -1,217 +0,0 @@
-package net.trustie.webmagic.one;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Resource;
-
-import net.trustie.webmagic.extrator.CSSPathExtractor;
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.URLPatternExtractor;
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.utils.LongSleepHelper;
-import net.trustie.webmagic.utils.MyRandomer;
-import net.trustie.webmagic.utils.SubSite;
-
-import org.apache.log4j.Logger;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.downloader.HttpClientDownloader;
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-
-@Component
-public class Crawler {
- private HttpClientDownloader downloader = new HttpClientDownloader();
- private Configure conf;
- private Set acceptStatCode;
- private String domain;
- private Site site;
- private MyTask task = new MyTask();
- Logger logger = Logger.getLogger(this.getClass());
- Map> lastPages = new HashMap>();
-
- @Resource
- private DetailHtmlDao detailHtmlDao;
-
- @Resource
- private ErrorPageDao errorPageDao;
- private String detailHtmlTableName;
- private String errorPageTableName;
-
- public static void main(String args[]){
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final Crawler htmlCrawler = applicationContext
- .getBean(Crawler.class);
- htmlCrawler.crawl(configureName);
- System.out.println("this is in th crawler");
- }
-
-
- private void crawl(String configureName) {
- // TODO Auto-generated method stub
- init(configureName);
-
- Extractor extractor = null;
- if ("URLPattern".equals(conf.getExtractMethod())) {
- extractor = new URLPatternExtractor(conf.getUrlPost());
- } else if ("CSSPath".equals(conf.getExtractMethod())) {
- extractor = new CSSPathExtractor(conf.getUrlPost());
- } else {
- logger.error("extract method error!!!");
- }
-
- int subSiteNum = this.conf.getSubSiteNum();
- logger.info("该站点共有" + subSiteNum + "个子站点");
-
- SubSite subSite;
-
- while(true){
- int doneNum = 0;//记录已完成镜像爬取的子站点个数
- while(doneNum++ < subSiteNum){
- //conf 负责获得这次处理的站点
- subSite = this.conf.nexSite();
-
- logger.info("现在爬取子站点 " + subSite.getName());
-
- Page page = downloader.download(new Request(
- subSite.getPrefixUrl()
- + subSite.getStartPageIndex()
- + subSite.getPostfixUrl()), task);
-
- Html html = new Html(UrlUtils.fixAllRelativeHrefs(
- page.getRawText(),this.conf.getFixAllRelativeHrefs()));
-
- Set nowUrls = extractor.extract(html);
- String pageName = this.domain + subSite.getName();
- Set lastUrls = lastPages.get(pageName);
-
- Set newUrls = new HashSet();
- if(lastUrls == null){
- logger.info("首爬 - 全新");
- lastPages.put(pageName,nowUrls);
- newUrls = nowUrls;
- }else{
- for(String url : nowUrls){
- if(!lastUrls.contains(url)){
- newUrls.add(url);
- logger.info("new url : " + url);
- }
- }
- }
- lastPages.put(pageName,nowUrls);
- if(newUrls.isEmpty()){
- logger.info("no new urls");
- }
-
-
- for(String url : newUrls){
- page = downloader.download(new Request(url), task);
- DetailHtml detailHtml = new DetailHtml();
- detailHtml.setStatusCode(page.getStatusCode());
- detailHtml.setHtml(page.getRawText());
- detailHtml.setUrl(page.getRequest().getUrl());
- if (page.getStatusCode() == 200) {
- detailHtmlDao.insertDetailHtml(detailHtml ,
- this.detailHtmlTableName);
- logger.info("save page " + page.getUrl() + " success");
- } else {
- errorPageDao.insertPage(this.errorPageTableName,
- detailHtml);
- logger.info("save error page " + detailHtml.getUrl() + "error"
- + detailHtml.getStatusCode() + " success!");
- }
- int time = MyRandomer.getRandomNum(conf.getMinInterval(),
- conf.getMaxInterval());
- try {
- logger.info("sleep " + time / 1000 + " s ...");
- Thread.sleep(time);
-
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
- try {
- int time = MyRandomer.getRandomNum(conf.getMinlongSleepTime(),conf.getMaxlongSleepTime());
- logger.info("next round will start in " + time / 1000 + " s...");
- Thread.sleep(time);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
-
-
-
- }
-
-
- private void init(String configureName) {
- // TODO Auto-generated method stub
- this.conf = new Configure(configureName);
- this.acceptStatCode = this.initAcceptStatCode();
- this.domain = conf.getDomain();
- this.site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- task.site = site;
-
- this.detailHtmlTableName = this.domain + "_html_detail";
- this.errorPageTableName = this.domain + "_error_page";
- }
-
-
- private Set initAcceptStatCode() {
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
- return acceptStatCode;
- }
-}
-
-class MyTask implements Task{
- Site site;
- @Override
- public String getUUID() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Site getSite() {
- // TODO Auto-generated method stub
- return site;
- }
-
- @Override
- public int getThreadAlive() {
- // TODO Auto-generated method stub
- return 0;
- }
-}
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java
deleted file mode 100644
index 74cf82f45..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java
+++ /dev/null
@@ -1,230 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.URL;
-import net.trustie.webmagic.utils.DBPipeline;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.LongSleepHelper;
-import net.trustie.webmagic.utils.Protector;
-import net.trustie.webmagic.utils.TableHelper;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.apache.log4j.xml.DOMConfigurator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.pipeline.Pipeline;
-import us.codecraft.webmagic.scheduler.QueueScheduler;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-import us.codecraft.webmagic.scheduler.component.BloomFilterDuplicateRemover;
-
-import javax.annotation.Resource;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Created by gyiang on 2014/11/15.
- */
-@Component
-public class DetailHtmlCrawler extends Thread{
- Logger logger = Logger.getLogger(this.getClass());
- private int batchSize = 10;
- @Resource
- private URLDao urlDao;
- @Resource
- private PointerDAO pointerDao;
-
- @Qualifier("detailHtmlPipeline")
- @Autowired
- private DBPipeline pipeline;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String urlTableName = "";
- private String urlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
-
- private String detailHtmlTableName = "";
- private String detailHtmlTableSql = "";
- private Configure conf = null;
-
- Spider spider;
- Site site;
- private void init(String configureName) {
- conf = new Configure(configureName);
- domain = conf.getDomain();
- detailHtmlTableName = domain + "_html_detail";
- urlTableName = domain + "_url";
- errorPageTableName = domain + "_error_page";
- initSql();
- initTable();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.urlTableSql = FileReader.readFile("./sql/url.sql");
- this.detailHtmlTableSql = FileReader.readFile("./sql/detail_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
- initOneTable(this.detailHtmlTableName, this.detailHtmlTableSql);
- initOneTable(this.urlTableName, this.urlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- public void crawl(String configureName) throws InterruptedException {
- // Configure conf = new Configure(configureName);
- this.init(configureName);
- // 每一批取线程数个url
- batchSize = conf.getThreadNum();
- Set acceptStatCode = new HashSet();
- //pipeline.setTableName(this.detailHtmlTableName);
- pipeline.setDetailHtmlTableName(this.detailHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
-
- // List proxies = Utils.getProxies(conf);
- site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- // .setHttpProxyPool(proxies);
- // int offset = 0;// conf.getOffset();
- // int limitLine = conf.getLimitLine();
-
-
- int startId = this.getPointer(this.urlTableName);
- List urls = new ArrayList();
- List urlList = new ArrayList();
-
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
- int noListHtmlTimes = 0;//record how many times that read no url
- while (true) {
- //lzx
- /*spider = Spider
- .create(new DetailHtmlProcessor(site))
- .addPipeline(pipeline).setScheduler(new SimpleScheduler()).thread(conf.getThreadNum());*/
- // int endId = startId + this.batchSize;
- //urls = urlDao.getBatch(this.urlTableName, startId, this.batchSize);
- urls = urlDao.getBatch(this.urlTableName, startId, 1);
- //urlList = new ArrayList();
- System.out.println("urls: " + urls.size());
- for (URL url : urls) {
- urlList.add(url.getUrl());
- startId = url.getId();
- }
- if (urlList.size() == 0) {
- System.out.println("no url in queue--->Thread.sleep()");
- noListHtmlTimes++;
- if(noListHtmlTimes >= 10){
- break;
- }
- Thread.sleep(conf.getNoUrlWaitTime());
- continue;
- }else{
- noListHtmlTimes = 0;
- }
-
- //lzx
- /*spider.startUrls(urlList);
- spider.run();*/
- renewSpider();
- spider.startUrls(urlList);
- spider.run();
-
- pointerDao.updatePointer(this.pointersTableName, this.urlTableName,
- startId);
- urls.clear();
- urlList.clear();
-
- // sleep
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000
- + "s");
- Thread.sleep(longSleepTime/5);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 /5 + "s");
- Thread.sleep(conf.getSleepTimeSpider()/5);
- }
-
- }
- }
-
- private int getPointer(String tableName) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName, 0);
- }
- return pointer;
-
- }
- private void renewSpider() {
-
- // .setHttpProxyPool(proxies);
- // .setCharset("UTF-8");
- Protector.setRandomArgs(site,conf);
- this.spider = Spider
- .create(new DetailHtmlProcessor(site))
- .addPipeline(this.pipeline)
-
- // .addUrl(conf.getStartUrl())
- // .setScheduler(new QueueScheduler().setDuplicateRemover(new
- // BloomFilterDuplicateRemover(10000))
- .thread(conf.getThreadNum())
- .setScheduler(new SimpleScheduler());
- }
- public static void main(String[] args) throws InterruptedException {
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final DetailHtmlCrawler htmlCrawler = applicationContext
- .getBean(DetailHtmlCrawler.class);
- htmlCrawler.crawl(configureName);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java
deleted file mode 100644
index 7ba947bca..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.utils.DBPipeline;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-
-import java.util.List;
-
-/**
- * Created by gyiang on 2014/11/16.
- */
-@Component("detailHtmlPipeline")
-public class DetailHtmlPipeline extends DBPipeline {
- Logger logger = Logger.getLogger(DetailHtmlPipeline.class);
- @Resource
- private DetailHtmlDao detailHtmlDao;
-
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- DetailHtml detailHtml = resultItems.get("model");
-
- try {
- if (detailHtml.getStatusCode() == 200) {
- detailHtmlDao.insertDetailHtml(detailHtml,
- this.getDetailHtmlTableName());
- logger.info("save page " + detailHtml.getUrl() + " success");
- } else {
- errorPageDao.insertPage(this.getErrorPageTableName(),
- detailHtml);
- logger.info("save error page " + detailHtml.getUrl() + "error"
- + detailHtml.getStatusCode() + " success!");
- }
- } catch (Exception e) {
- e.printStackTrace();
- // Logger logger = Logger.getLogger(MySqlPipelinePost.class);
- logger.error("error on Pipeline,when:" + detailHtml.getUrl());
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java
deleted file mode 100644
index d7b96bb2d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package net.trustie.webmagic.one;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.commons.codec.digest.DigestUtils;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-public class DetailHtmlProcessor implements PageProcessor {
- private Site site;
-
- // private String listHtmlTName;// listHtmlTableName
- // private String reList;
-
- public DetailHtmlProcessor(Site site) {
- this.site = site;
- }
-
- @Override
- public void process(Page page) {
-
- DetailHtml detailHtml = new DetailHtml();
- detailHtml.setStatusCode(page.getStatusCode());
- String html = page.getRawText();
- detailHtml.setHtml(html);
- String url = page.getRequest().getUrl();
- detailHtml.setUrl(url);
- if (url != null) {
- detailHtml.setUrlMd5(DigestUtils.md5Hex(page.getUrl().get()));
- }
- detailHtml.setType("Detail");
-
- detailHtml.setCrawledTime(Utils.getNow());
- if (html != null) {
- detailHtml.setPageMd5(DigestUtils.md5Hex(html));
- }
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", detailHtml);
- // page.putField("listTableName", listTableName);
- }
-
- @Override
- public Site getSite() {
- return site;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlSubProcessor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlSubProcessor.java
deleted file mode 100644
index 01dc7a4cc..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlSubProcessor.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.log4j.Logger;
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-import us.codecraft.webmagic.selector.PlainText;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class DetailHtmlSubProcessor implements SubPageProcessor {
-
- private Logger logger = Logger.getLogger(DetailHtmlSubProcessor.class);
-
- // public static final String URL_POST =
- // "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";
-
- @Override
- public MatchOther processPage(Page page) {
-
-// SimpleDateFormat timestampFormat = new SimpleDateFormat(
-// "yyyyMMddHHmmss");
- SimpleDateFormat crawledTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- Date date = new Date();
- //String timestamp = timestampFormat.format(date);
- String crawledTime = crawledTimeFormat.format(date);
-
- DetailHtml detailHtml = new DetailHtml();
- if (page.getStatusCode() == 404 || page.getStatusCode() == 503) {
- detailHtml.setStatusCode(page.getStatusCode());
- //detailHtml.setTimestamp(page.getStatusCode() + "");
- page.setSkip(true);// 跳过这个页面
- } else {
- //detailHtml.setStatusCode(statusCode);
- }
-
- detailHtml.setHtml(page.getRawText().toString());
- detailHtml.setUrl(page.getUrl().get());
- detailHtml.setUrlMd5(DigestUtils.md5Hex(page.getUrl().get()));
- detailHtml.setType("Detail");
-
- /*
- * if(page.getUrl().regex(reList).match()) { htmlModel.setType("list");
- * }else if(page.getUrl().regex(rePost).match()) {
- * htmlModel.setType("post"); }else {
- * logger.error("unknown RE,current url:"+page.getUrl().get()); }
- */
-
- detailHtml.setCrawledTime(crawledTime);
- detailHtml.setPageMd5(DigestUtils.md5Hex(page.getRawText().toString()));
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", detailHtml);
- //page.putField("listTableName", listTableName);
-
- return MatchOther.YES;
- }
-
-// public DetailHtmlSubProcessor(String postTableName, String listTableName) {
-// this.postTableName = postTableName;
-// this.listTableName = listTableName;
-// }
-
- @Override
- public boolean match(Request page) {
- /*
- * if (new PlainText(page.getUrl()).regex(URL_POST).match()){ return
- * true; }else { return false; }
- */
- return true;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageCrawler.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageCrawler.java
deleted file mode 100644
index dbb2dd103..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageCrawler.java
+++ /dev/null
@@ -1,197 +0,0 @@
-package net.trustie.webmagic.one;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Resource;
-
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.URL;
-import net.trustie.webmagic.utils.DBPipeline;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.TableHelper;
-
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-
-/**
- * Created by gyiang on 2014/11/15.
- */
-@Component
-public class ErrorPageCrawler {
- Logger logger = Logger.getLogger(this.getClass());
- private int batchSize = 10;
- @Resource
- private ErrorPageDao errorPageDao;
- @Resource
- private PointerDAO pointerDao;
-
- @Qualifier("errorPagePipeline")
- @Autowired
- private DBPipeline pipeline;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String urlTableName = "";
- private String urlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
-
- private String listHtmlTableName = "";
- private String listHtmlTableSql = "";
-
- private String detailHtmlTableName = "";
- private String detailHtmlTableSql = "";
- private Configure conf = null;
-
- private void init(String configureName) {
- conf = new Configure(configureName);
- domain = conf.getDomain();
- listHtmlTableName = domain + "_html_list";
- detailHtmlTableName = domain + "_html_detail";
- urlTableName = domain + "_url";
- errorPageTableName = domain + "_error_page";
- batchSize = conf.getThreadNum();
- pipeline.setListHtmlTableName(this.listHtmlTableName);
- pipeline.setDetailHtmlTableName(this.detailHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
- initSql();
- initTable();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.urlTableSql = FileReader.readFile("./sql/url.sql");
- this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
- this.detailHtmlTableSql = FileReader.readFile("./sql/detail_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
- initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
- initOneTable(this.detailHtmlTableName, this.detailHtmlTableSql);
- initOneTable(this.urlTableName, this.urlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- public void crawl(String configureName) throws InterruptedException {
- // Configure conf = new Configure(configureName);
- this.init(configureName);
- // 每一批取线程数个url
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
-
- // List proxies = Utils.getProxies(conf);
- Site site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- // .setHttpProxyPool(proxies);
- // int offset = 0;// conf.getOffset();
- // int limitLine = conf.getLimitLine();
- Spider spider = Spider.create(new ErrorPageProcessor(site))
- .addPipeline(pipeline).setScheduler(new SimpleScheduler())
- .thread(conf.getThreadNum());
-
- int startId = this.getPointer(this.errorPageTableName);
- List urls = new ArrayList();
- Request[] urlArray = null;
- Request request = null;
- Map maps = new HashMap();
-
- while (true) {
- // int endId = startId + this.batchSize;
- urls = errorPageDao.getBatch(this.errorPageTableName, startId,
- this.batchSize);
- urlArray = new Request[urls.size()];
- int i = 0;
- for (URL url : urls) {
- request = new Request(url.getUrl());
- maps.put("type", url.getType());
- request.setExtras(maps);
- urlArray[i] = request;
- startId = url.getId();
- i++;
- }
- i = 0;
- if (urlArray.length == 0) {
- System.out.println("no error url in queue--->Thread.sleep()");
- Thread.sleep(conf.getNoUrlWaitTime());
- continue;
- }
-
- spider.addRequest(urlArray);
- spider.run();
-
- pointerDao.updatePointer(this.pointersTableName,
- this.errorPageTableName, startId);
- urls.clear();
- // sleep
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- }
- }
-
- private int getPointer(String tableName) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName, 0);
- }
- return pointer;
-
- }
-
- public static void main(String[] args) throws InterruptedException {
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final ErrorPageCrawler errorPageCrawler = applicationContext
- .getBean(ErrorPageCrawler.class);
- errorPageCrawler.crawl(configureName);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPagePipeline.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPagePipeline.java
deleted file mode 100644
index 666ae6f82..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPagePipeline.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.AbstractHtml;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.DBPipeline;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-
-import java.util.List;
-
-/**
- * Created by gyiang on 2014/11/16.
- */
-@Component("errorPagePipeline")
-public class ErrorPagePipeline extends DBPipeline {
- Logger logger = Logger.getLogger(this.getClass());
- @Resource
- private DetailHtmlDao detailHtmlDao;
-
- @Resource
- private ListHtmlDao listHtmlDao;
-
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- AbstractHtml html = resultItems.get("model");
-
- try {
- if (html.getStatusCode() == 200) {
- if ("List".equals(html.getType())) {
- listHtmlDao.insertListHtml((ListHtml) html,
- this.getListHtmlTableName());
- } else if ("Detail".equals(html.getType())) {
- detailHtmlDao.insertDetailHtml((DetailHtml) html,
- this.getDetailHtmlTableName());
- }
- logger.info("save page " + html.getUrl() + " success!");
- } else {
- errorPageDao.insertPage(this.getErrorPageTableName(), html);
- logger.info("save error page " + html.getUrl() + "error"
- + html.getStatusCode() + " success!");
- }
- } catch (Exception e) {
- e.printStackTrace();
- // Logger logger = Logger.getLogger(MySqlPipelinePost.class);
- logger.error("error on Pipeline,when:" + html.getUrl());
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageProcessor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageProcessor.java
deleted file mode 100644
index e7528eb29..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageProcessor.java
+++ /dev/null
@@ -1,85 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.log4j.Logger;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-/**
- * Created by gyiang on 2014/12/19.
- */
-public class ErrorPageProcessor implements PageProcessor {
- Logger logger = Logger.getLogger(this.getClass());
- private Site site;
-
- public ErrorPageProcessor(Site site) {
- this.site = site;
- }
-
- @Override
- public void process(Page page) {
-
- if ("List".equals((String) (page.getRequest().getExtra("type")))) {
- constructListHtml(page);
- } else if ("Detail"
- .equals((String) (page.getRequest().getExtra("type")))) {
- constructDetailHtml(page);
- } else {
- logger.info("errorPage type error :" + page.getRequest().getUrl());
- }
- }
-
- private void constructListHtml(Page page) {
- ListHtml listHtml = new ListHtml();
- listHtml.setStatusCode(page.getStatusCode());
- String html = "";
- html = page.getRawText();
- listHtml.setHtml(html);
-
- listHtml.setCrawledTime(Utils.getNow());
- String url = page.getUrl().get();
- listHtml.setUrl(url);
- listHtml.setUrlMd5(DigestUtils.md5Hex(url));
- listHtml.setPageMd5(DigestUtils.md5Hex(html));// 用于确定是已经抽过
- listHtml.setType("List");
- page.putField("model", listHtml);
- }
-
- private void constructDetailHtml(Page page) {
- DetailHtml detailHtml = new DetailHtml();
-
- detailHtml.setStatusCode(page.getStatusCode());
-
- detailHtml.setHtml(page.getRawText().toString());
- detailHtml.setUrl(page.getUrl().get());
- detailHtml.setUrlMd5(DigestUtils.md5Hex(page.getUrl().get()));
- detailHtml.setType("Detail");
-
- /*
- * if(page.getUrl().regex(reList).match()) { htmlModel.setType("list");
- * }else if(page.getUrl().regex(rePost).match()) {
- * htmlModel.setType("post"); }else {
- * logger.error("unknown RE,current url:"+page.getUrl().get()); }
- */
-
- detailHtml.setCrawledTime(Utils.getNow());
- detailHtml.setPageMd5(DigestUtils.md5Hex(page.getRawText().toString()));
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", detailHtml);
- }
-
- @Override
- public Site getSite() {
- return site;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlProcessor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlProcessor.java
deleted file mode 100644
index ec3ddcf94..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlProcessor.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.HtmlSubProcessorList;
-import net.trustie.webmagic.one.DetailHtmlSubProcessor;
-import net.trustie.webmagic.utils.LoadConf;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class HtmlProcessor extends CompositePageProcessor {
-
- private Site site = null;
- private String regexUrl;
- private String sourceRegion;
-
- public HtmlProcessor(Site site, String regexUrlList, String regexUrlPost,
- String listTableName) {
-
- super(site);
- this.site = site;
- // super.addSubPageProcessor(new HtmlSubProcessorPost());
- super.addSubPageProcessor(new HtmlSubProcessorList(regexUrlList,
- regexUrlPost, listTableName));
- }
-
- /*
- * @Override public void process(Page page) {
- * page.addTargetRequests(page.getHtml
- * ().xpath(sourceRegion).links().regex(regexUrl).all());
- *
- * }
- */
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlSubProcessorList.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlSubProcessorList.java
deleted file mode 100644
index 0911bf8f5..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlSubProcessorList.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package net.trustie.webmagic.one;
-
-import org.apache.log4j.Logger;
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-
-import java.util.List;
-
-/**
- * Created by gan on 2014/11/15. Notified by gan on 2014/11/
- */
-public class HtmlSubProcessorList implements SubPageProcessor {
- /*
- * 最初使用static final方法 public static final String URL_LIST; public static
- * final String URL_POST ;
- */
- private String urlList;
- private String urlPost;
- private String listTableName;
-
- public HtmlSubProcessorList(String urlList, String urlPost,
- String listTableName) {
- this.urlList = urlList;
- this.urlPost = urlPost;
- this.listTableName = listTableName;
- }
-
- @Override
- public MatchOther processPage(Page page) {
-
- // 加入爬取队列,策略改变后,变成存入数据库
- /*
- * page.addTargetRequests(page.getHtml().xpath(
- * "//div[@class=\"articleList\"]").links().regex(urlPost).all());
- * page.addTargetRequests(page.getHtml().links().regex(urlList).all());
- * page.getTargetRequests()
- */
- // 列表页链接加入,发现新的列表页
-
- // System.out.println(page.getRawText());
- page.addTargetRequests(page.getHtml().links().regex(urlList).all());
- // 获取post链接
- List targetUrl = page.getHtml().links().regex(urlPost).all();
- // 列表页的url也存入,去爬
- targetUrl.add(page.getUrl().get());
-
- // 交给pipeline
- page.putField("findUrl", targetUrl);
- page.putField("listTableName", listTableName);
-
- // 只取列表页,跳过持久化过程
- // page.setSkip(false);
- return MatchOther.YES;
- }
-
- @Override
- public boolean match(Request page) {
- return PatternUrl.match(page, urlList);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java
deleted file mode 100644
index 947fed662..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java
+++ /dev/null
@@ -1,499 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.CreateTableDAO;
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.utils.DBPipeline;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.LongSleepHelper;
-import net.trustie.webmagic.utils.MyRandomer;
-import net.trustie.webmagic.utils.OrderSpider;
-import net.trustie.webmagic.utils.Protector;
-import net.trustie.webmagic.utils.SubSite;
-import net.trustie.webmagic.utils.TableHelper;
-import net.trustie.webmagic.utils.UserAgentPool;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.http.HttpHost;
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.apache.log4j.xml.DOMConfigurator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.monitor.SpiderMonitor;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.pipeline.Pipeline;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-
-import javax.annotation.Resource;
-import javax.management.JMException;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-import java.io.File;
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-
-/**
- * Created by Administrator on 2014/12/19.
- */
-@Component
-public class ListHtmlCrawler {
- Logger logger = Logger.getLogger(this.getClass());
- @Qualifier("listHtmlPipeline")
- @Autowired
- private DBPipeline pipeline;
- @Resource
- private PointerDAO pointerDao;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String listHtmlTableName = "";
- private String listHtmlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
- private String pageIndexName = "";
- private String modeColName = "";
- // 记录列表页爬取位置的字段名
-
- private int startIndex = 0;
- private int endIndex = 0;
- private int mode;
-
- private Configure conf = null;
- private Set acceptStatCode = null;
- private Site site = null;
- private Spider spider = null;
- private Random random = new Random(66);
- private void init(String configureName) {
- this.conf = new Configure(configureName);
- this.acceptStatCode = this.initAcceptStatCode();
- this.domain = conf.getDomain();
- this.listHtmlTableName = this.domain + "_html_list";
- this.errorPageTableName = this.domain + "_error_page";
- // 轮换注释掉 this.pageIndexName = this.domain + "_page_index";
- this.modeColName = this.domain + "_mode";
-
-
- initSql();
- // 初始化表
- initTable();
-
- // 轮换注释掉 startIndex = this.getPointer(pageIndexName,this.conf.getStartPageIndex());
- mode = this.getMode(this.modeColName, this.conf.getMode());
-
- pipeline.setListHtmlTableName(this.listHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
-
- // List proxies = Utils.getProxies(conf);
- this.site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- renewSpider();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
- initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- // 取得指针
- private int getPointer(String tableName, int defaultValue) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- return pointer + 1;
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName,
- defaultValue - 1);
- }
- return defaultValue;
-
- }
-
- private int getMode(String tableName, int defaultValue) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- return pointer;
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName,
- defaultValue);
- }
- return defaultValue;
- }
-
- public void crawl(String configureName) {
- this.init(configureName);
-
- this.acceptStatCode = this.initAcceptStatCode();
-// pipeline.setTableName(this.listHtmlTableName);
-// pipeline.setErrorPageTableName(this.errorPageTableName);
-
- this.startSpider();
- }
-
- private List getOrderedUrl(String purl, String surl, int per,
- int incIndex, int startIndex) {
- List urls = new ArrayList();
- for (int g = startIndex; g < startIndex + per; g++) {
- int pageindex = g + incIndex;
- String url = purl + pageindex + surl;
- urls.add(url);
- // System.out.println(url);
- }
- return urls;
- }
-
- private void startSpider() {
- // List urls = new ArrayList();
- if (this.mode == 0) { // 镜像模式
- this.mirrorModeCrawl();
- logger.info("change to increment mode!!!");
- pointerDao.updatePointer(this.pointersTableName, this.modeColName,
- 1);
-
- this.reInit();
- this.incrementModeCrawl();
- } else if (this.mode == 1) { // 增量模式
- this.incrementModeCrawl();
- } else {
- logger.info("mode error!!!");
- }
- }
-
- private void reInit() {
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, this.conf.getStartPageIndex() - 1);//增量结束归起始页 ;
- //this.startIndex = this.conf.getStartPageIndex();
- }
-
-
-
-
- // 镜像模式爬取
- private void mirrorModeCrawl() {
-
-
-
- //star modify for 轮换
- int subSiteNum = this.conf.getSubSiteNum();
-
- logger.info("该站点共有" + subSiteNum + "个子站点");
- int doneNum = 0;//记录已完成镜像爬取的子站点个数
- int processNum = 10;//对每个子站点一次处理多少个页面
- while(doneNum < subSiteNum){
-
- //conf 负责获得这次处理的站点
- SubSite subSite = this.conf.nexSite();
- if(subSite.getName().equals(""))
- this.pageIndexName = this.domain +"_page_index";
- else
- this.pageIndexName = this.domain + "_" + subSite.getName()+ "_page_index";
-
- startIndex = this.getPointer(pageIndexName,this.conf.getStartPageIndex());
-
- List urls = new ArrayList();
- this.endIndex = this.conf.getEndInPageIndex();
-
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
- int j =0;
- int i = 0;
- for ( i = this.startIndex;( i <= this.endIndex ) && (j < processNum); i += conf.getPageF(),j++) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
-
- //对爬虫的参数从新进行随机化设置
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000 / 5
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider()/5);
- }
- } catch (InterruptedException e) {
-
- e.printStackTrace();
- }
- }
-
- if(i > this.endIndex)
- doneNum++;//此站点已经处理完毕
-
-
- }
-
-
-
- //end of star modify 轮换
- /*List urls = new ArrayList();
- this.endIndex = this.conf.getEndInPageIndex();
-
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
-
- for (int i = this.startIndex; i <= this.endIndex; i += conf.getPageF()) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
-
- //对爬虫的参数从新进行随机化设置
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- }
- } catch (InterruptedException e) {
-
- e.printStackTrace();
- }
- }*/
- }
-
- private void renewSpider() {
-
- // .setHttpProxyPool(proxies);
- // .setCharset("UTF-8");
- Protector.setRandomArgs(site,conf);
- this.spider = Spider
- .create(new ListHtmlProcessor(site))
- .addPipeline(this.pipeline)
-
- // .addUrl(conf.getStartUrl())
- // .setScheduler(new QueueScheduler().setDuplicateRemover(new
- // BloomFilterDuplicateRemover(10000))
- .thread(conf.getThreadNum())
- .setScheduler(new SimpleScheduler());
- }
-
- // 镜像模式爬取
- private void incrementModeCrawl() {
-
- List urls = new ArrayList();
-
-
-
- //while (true) {
-
- //star modify for 轮换
- int subSiteNum = this.conf.getSubSiteNum();
- logger.info("该站点共有" + subSiteNum + "个子站点");
- int doneNum = 0;//记录已完成镜像爬取的子站点个数
- int processNum = 10;//对每个子站点一次处理多少个页面
- while(doneNum < subSiteNum){
- //conf 负责获得这次处理的站点
- SubSite subSite = this.conf.nexSite();
-
- if(subSite.getName().equals(""))
- this.pageIndexName = this.domain +"_page_index";
- else
- this.pageIndexName = this.domain + "_" + subSite.getName()+ "_page_index";
-
- startIndex = this.getPointer(pageIndexName,this.conf.getStartPageIndex());
-
- this.endIndex = this.conf.getIncrementalPages();
- //System.out.println("增量爬取" + startIndex + "~" + endIndex );
- if(startIndex >= endIndex)
- startIndex = this.conf.getStartPageIndex();//防止异常退出后,转换为增量或增量结束后指针没有来得及更新为起始页
- //System.out.println("增量爬取" + startIndex + "~" + endIndex );
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
- int j =0;
- int i = 0;
- for (i = this.startIndex;( i <= this.endIndex ) && (j < processNum); i += conf.getPageF(),j++) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
-
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000/5
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 /5
- + "s");
-
- Thread.sleep(conf.getSleepTimeSpider()/5);
- }
-
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- if(i > this.endIndex){
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, this.conf.getStartPageIndex() - 1);//增量结束归起始页
- doneNum++;//此站点已经处理完毕
- }
-
- }
-
-// this.reInit();
-// logger.info("increment finish!!!");
-// logger.info("sleep "+conf.getIncrementSleepTime()+"s");
-// try {
-// Thread.sleep(conf.getIncrementSleepTime()*1000);
-// } catch (InterruptedException e) {
-// // TODO Auto-generated catch block
-// e.printStackTrace();
-// }
-
- //this.reInit();
- //}
-
-
- }
-
- private void incrementModeCrawlHelper(){
- List urls = new ArrayList();
- this.endIndex = this.conf.getIncrementalPages();
- // System.out.println(startIndex);
- // System.out.println(endIndex);
-
-
- //压力分散做法行不通,因为列表页的内容是不断的在变化的,此时的第一页和一个小时前的第一页已经大不一样了
- //如果一天爬取完,那平均多少秒爬一个
- //int baseTime = (24 * 60 * 60 * 1000) / this.endIndex;
- //设置爬虫新的随机区间,请求间隔不要太小,设置为一半,考虑到其他的消耗,也不要太长,设置为2/3
- //this.conf.setMinInterval(baseTime / 2);
- //this.conf.setMaxInterval(baseTime*5 / 6);
-
-
- // while (true) {
- for (int i = this.startIndex; i <= this.endIndex; i += conf.getPageF()) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000
- + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- logger.info("increment finish!!!");
- // logger.info("sleep "+conf.getIncrementSleepTime()+"s");
- /*try {
- Thread.sleep(conf.getIncrementSleepTime()*1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- */
- this.reInit();
- // }
- }
- private Set initAcceptStatCode() {
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
- return acceptStatCode;
- }
-
- public static void main(String[] args) throws InterruptedException {
- // DOMConfigurator.configure("log4j.xml");
- // System.err.println(new File("log4j.xml").getAbsolutePath());
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final ListHtmlCrawler htmlCrawler = applicationContext
- .getBean(ListHtmlCrawler.class);
- htmlCrawler.crawl(configureName);
- System.out.println("this is in th crawler");
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlPipeline.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlPipeline.java
deleted file mode 100644
index c9f05f59d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlPipeline.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.DBPipeline;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-
-/**
- * Created by ganyiang on 2014/12/19.
- */
-@Component("listHtmlPipeline")
-public class ListHtmlPipeline extends DBPipeline {
- Logger logger = Logger.getLogger(ListHtmlPipeline.class);
- @Resource
- private ListHtmlDao listHtmlDao;
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- ListHtml listHtml = resultItems.get("model");
- try {
- if (listHtml.getStatusCode() == 200) {
- listHtmlDao.insertListHtml(listHtml,
- this.getListHtmlTableName());
- logger.info("save page " + listHtml.getUrl());
- } else {
- errorPageDao.insertPage(this.getErrorPageTableName(), listHtml);
- logger.info("save error page " + listHtml.getUrl() + " error "
- + listHtml.getStatusCode()+" success!");
- }
- } catch (Exception e) {
- e.printStackTrace();
- // Logger logger = Logger.getLogger(MySqlPipelinePost.class);
- logger.error("error on Pipeline,when:" + listHtml.getUrl());
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java
deleted file mode 100644
index 20372efaf..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.commons.codec.digest.DigestUtils;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-/**
- * Created by gyiang on 2014/12/19.
- */
-public class ListHtmlProcessor implements PageProcessor {
-
- private Site site;
-
- public ListHtmlProcessor(Site site) {
- this.site = site;
- }
-
- @Override
- public void process(Page page) {
-
- ListHtml listHtml = new ListHtml();
-
- listHtml.setStatusCode(page.getStatusCode());
- String html = page.getRawText();
- listHtml.setHtml(html);
-
- listHtml.setCrawledTime(Utils.getNow());
- String url;// = page.getUrl().get();
- url = page.getRequest().getUrl();
- listHtml.setUrl(url);
- if (url != null) {
- listHtml.setUrlMd5(DigestUtils.md5Hex(url));
- }
- if (html != null) {
- listHtml.setPageMd5(DigestUtils.md5Hex(html));// 用于确定是已经抽过
- }
- listHtml.setType("List");
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", listHtml);
- }
-
- @Override
- public Site getSite() {
- return site;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MakeUp.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MakeUp.java
deleted file mode 100644
index 408e9745a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MakeUp.java
+++ /dev/null
@@ -1,183 +0,0 @@
-package net.trustie.webmagic.one;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Resource;
-
-import net.trustie.webmagic.extrator.CSSPathExtractor;
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.URLPatternExtractor;
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.utils.LongSleepHelper;
-import net.trustie.webmagic.utils.MyRandomer;
-import net.trustie.webmagic.utils.SubSite;
-
-import org.apache.log4j.Logger;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.downloader.HttpClientDownloader;
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-
-@Component
-public class MakeUp {
- private HttpClientDownloader downloader = new HttpClientDownloader();
- private Configure conf;
- private Set acceptStatCode;
- private String domain;
- private Site site;
- private MyTask2 task = new MyTask2();
- Logger logger = Logger.getLogger(this.getClass());
- Map> lastPages = new HashMap>();
-
- @Resource
- private DetailHtmlDao detailHtmlDao;
-
- @Resource
- private URLDao urlDao;
-
- @Resource
- private ErrorPageDao errorPageDao;
- private String detailHtmlTableName;
- private String errorPageTableName;
- private String urlTableName;
- SimpleDateFormat extractedTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- public static void main(String args[]){
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final MakeUp htmlCrawler = applicationContext
- .getBean(MakeUp.class);
- htmlCrawler.crawl(configureName);
- System.out.println("this is in th crawler");
- }
-
-
- private void crawl(String configureName) {
- // TODO Auto-generated method stub
- init(configureName);
-
- Extractor extractor = null;
- if ("URLPattern".equals(conf.getExtractMethod())) {
- extractor = new URLPatternExtractor(conf.getUrlPost());
- } else if ("CSSPath".equals(conf.getExtractMethod())) {
- extractor = new CSSPathExtractor(conf.getUrlPost());
- } else {
- logger.error("extract method error!!!");
- }
-
- int subSiteNum = this.conf.getSubSiteNum();
- logger.info("该站点共有" + subSiteNum + "个子站点");
-
- SubSite subSite;
-
-
- int doneNum = 0;//记录已完成镜像爬取的子站点个数
- while(doneNum ++< subSiteNum){
- //conf 负责获得这次处理的站点
- subSite = this.conf.nexSite();
-
- logger.info("现在爬取子站点 " + subSite.getName());
- int start = subSite.getStartPageIndex();
- int end = subSite.getEndInPageIndex();
- int j =0;
- int i =0;
- for( i = start;i <= end; i ++,j++){
- Page page = downloader.download(new Request(
- subSite.getPrefixUrl()
- + i
- + subSite.getPostfixUrl()), task);
-
- Html html = new Html(UrlUtils.fixAllRelativeHrefs(
- page.getRawText(),this.conf.getFixAllRelativeHrefs()));
-
- Set urls = extractor.extract(html);
-
- for(String url : urls){
- Date date = new Date();
- String extractedTime = extractedTimeFormat.format(date);
- urlDao.addURL(urlTableName, url, extractedTime);
- }
- try {
- Thread.sleep(MyRandomer.getRandomNum(15, 30) * 1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- }
-
- }
-
-
- private void init(String configureName) {
- // TODO Auto-generated method stub
- this.conf = new Configure(configureName);
- this.acceptStatCode = this.initAcceptStatCode();
- this.domain = conf.getDomain();
- this.site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- task.site = site;
-
- this.detailHtmlTableName = this.domain + "_html_detail";
- this.errorPageTableName = this.domain + "_error_page";
- this.urlTableName = this.domain + "_url";
- }
-
-
- private Set initAcceptStatCode() {
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
- return acceptStatCode;
- }
-}
-
-class MyTask2 implements Task{
- Site site;
- @Override
- public String getUUID() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public Site getSite() {
- // TODO Auto-generated method stub
- return site;
- }
-
- @Override
- public int getThreadAlive() {
- // TODO Auto-generated method stub
- return 0;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MianThread.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MianThread.java
deleted file mode 100644
index 8c215a06b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MianThread.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package net.trustie.webmagic.one;
-
-public class MianThread {
-
- public static void main(String[] args) throws InterruptedException {
- UrlExtractor url=new UrlExtractor(args[0]);
- url.start();
-
-
-
- Thread.sleep(100000);
- url.destroy();
-
-
-
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MySqlPipelineList.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MySqlPipelineList.java
deleted file mode 100644
index e6be3c981..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MySqlPipelineList.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import org.springframework.stereotype.Component;
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-
-/**
- * Created by gyiang on 2014/11/16.
- */
-@Component("MySqlPipelineList")
-public class MySqlPipelineList implements Pipeline {
- @Resource
- private URLDao htmlTargetUrlDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- List l = (List) resultItems.get("findUrl");
- // 爬取时间戳
- SimpleDateFormat bartDateFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- String crawlerTime = bartDateFormat.format(new Date());
- for (String urlPost : l) {
- htmlTargetUrlDao.add(urlPost, crawlerTime,
- resultItems.get("listTableName").toString());
- }
-
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/PatternUrl.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/PatternUrl.java
deleted file mode 100644
index 1b7fb89f3..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/PatternUrl.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package net.trustie.webmagic.one;
-
-import us.codecraft.webmagic.Request;
-
-import java.util.regex.Pattern;
-
-/**
- * Created by gyiang on 2014/11/19.
- */
-public class PatternUrl {
- protected String pattern;
- private Pattern patternCompiled;
-
- public static boolean match(Request request, String pattern) {
- return Pattern.compile(pattern).matcher(request.getUrl()).matches();
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/UrlExtractor.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/UrlExtractor.java
deleted file mode 100644
index 708d9d831..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/UrlExtractor.java
+++ /dev/null
@@ -1,265 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.extrator.CSSPathExtractor;
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.URLPatternExtractor;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.TableHelper;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.apache.log4j.xml.DOMConfigurator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.model.annotation.ExtractByUrl;
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-
-import javax.annotation.Resource;
-
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Created by Administrator on 2014/12/21.
- */
-@Component
-public class UrlExtractor extends Thread {
- Logger logger = Logger.getLogger(this.getClass());
- private static int batchSize = 10;
- @Resource
- private ListHtmlDao listHtmlDao;
- @Resource
- private PointerDAO pointerDao;
- @Resource
- private URLDao urlDao;
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String urlTableName = "";
- private String urlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
-
- private String listHtmlTableName = "";
- private String listHtmlTableSql = "";
- private Configure conf = null;
-
- private void init(String configureName) {
- conf = new Configure(configureName);
- domain = conf.getDomain();
- listHtmlTableName = domain + "_html_list";
- urlTableName = domain + "_url";
- errorPageTableName = domain + "_error_page";
- initSql();
- initTable();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.urlTableSql = FileReader.readFile("./sql/url.sql");
- this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
- initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
- initOneTable(this.urlTableName, this.urlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- public void exec(String configureName) throws InterruptedException {
- this.init(configureName);
- int startId = this.getPointer(listHtmlTableName);
- List list = new ArrayList();
- Set urlSet = new HashSet();
-
- Extractor extractor = null;
- if ("URLPattern".equals(conf.getExtractMethod())) {
- extractor = new URLPatternExtractor(conf.getUrlPost());
- } else if ("CSSPath".equals(conf.getExtractMethod())) {
- extractor = new CSSPathExtractor(conf.getUrlPost());
- } else {
- logger.error("extract method error!!!");
- }
- SimpleDateFormat extractedTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- Date date = null;
- String extractedTime = "";
- Html html = null;
- String fixRelativeHref = conf.getFixAllRelativeHrefs();
- int noListHtmlTimes = 0;//record how many times that read no listhtml
- while(true) {
- //Thread.sleep(100);
- // int startId = start+UrlExtractor.batchSize;
- // int endId = startId + UrlExtractor.batchSize;
- list = listHtmlDao.getBatch(listHtmlTableName, startId,
- UrlExtractor.batchSize);
- if (list.size() == 0) {
- System.out.println("no listHtml in queue--->Thread.sleep()");
- noListHtmlTimes++;
- if(noListHtmlTimes >= 10){
- break;
- }
- Thread.sleep(conf.getNoUrlWaitTime());
- continue;
- }else{
- noListHtmlTimes = 0;
- }
- for (ListHtml listHtml : list) {
- if (listHtml == null || listHtml.getHtml() == "") {
- continue;// 跳过
- }
- // SimpleDateFormat timestampFormat = new SimpleDateFormat(
- // "yyyyMMddHHmmss");
- date = new Date();
- // String timestamp = timestampFormat.format(date);
- extractedTime = extractedTimeFormat.format(date);
-
- html = new Html(UrlUtils.fixAllRelativeHrefs(
- listHtml.getHtml(), fixRelativeHref));
-
- urlSet = extractor.extract(html);
-
- String pageMd5 = DigestUtils.md5Hex(listHtml.getHtml());// html.get()
- // same
- // rawText
- String error = null;
-
- try {
- for (String url : urlSet) {
- try {
- urlDao.addURL(urlTableName, url, extractedTime);
- } catch (Exception e) {
- logger.error(e.getMessage());
- error = "500";
- }
- }
- // 存完一次后标记
- if (error != null) {
- // pointerDao.updatePointer(conf.getListHtmlTableName(),
- // listHtml.getId());
- // listHtmlDao.timestamp(listHtmlTableName, pageMd5,
- // timestamp);
- } else {// 成功
- pointerDao.updatePointer(this.pointersTableName,
- listHtmlTableName, listHtml.getId());
- // listHtmlDao
- // .timestamp(listHtmlTableName, pageMd5, error);
- }
- } catch (Exception e) {
- logger.error(e.getMessage());
-
- }
-
- logger.info("extracted id = " + listHtml.getId() + " "
- + listHtml.getUrl());
-
- startId = listHtml.getId();
- urlSet.clear();
-
- }
- list.clear();
- // startId=
- }
- }
-
- private int getPointer(String tableName) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName, 0);
- }
- return pointer;
-
- }
-
- public static void main(String[] args) throws InterruptedException {
- // DOMConfigurator.configure("log4j.xml");
- // System.err.println(new File("log4j.xml").getAbsolutePath());
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final UrlExtractor urlExtractor = applicationContext
- .getBean(UrlExtractor.class);
- urlExtractor.exec(configureName);
- // urlExtrator.crawl(configureName);
- }
-
-
- private String args;
-
- public UrlExtractor( String args) {
- this.args=args;
- }
-
-
- public UrlExtractor( ) {
- this.args=args;
- }
-
-
-
-
- @Override
- public void run() {
- String configureName =args;
-
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final UrlExtractor urlExtractor = applicationContext
- .getBean(UrlExtractor.class);
- try {
- urlExtractor.exec(configureName);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
-
- }
-
-
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/CreateTableDAO.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/CreateTableDAO.java
deleted file mode 100644
index d3208144d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/CreateTableDAO.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-public interface CreateTableDAO {
- //判断表是否存在
- @Select("select 1 from ${tableName}")
- public List isTableExist(@Param("tableName") String tableName);
-
- @Update("create table if not exists ${table} ${fields}")
- public int createTable(@Param("table") String table, @Param("fields") String fields);
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java
deleted file mode 100644
index 723622ed3..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import java.util.List;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-
-import org.apache.ibatis.annotations.Delete;
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-
-/**
- * Created by Administrator on 2014/11/16.
- */
-public interface DetailHtmlDao {
-
- @Insert("insert into "
- + "${tableName}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{html},#{url},#{crawledTime},#{history},#{pageMd5},#{urlMd5})")
- public int add(DetailHtml detailHtml);
-
- @Insert("insert into "
- + "${table2Name}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{html},#{url},#{crawledTime},#{history},#{pageMd5},#{urlMd5})")
- public int addListHtml(DetailHtml detailHtml);
-
- @Delete("delete from ${tableName} where url=#{url}")
- public int delete404(@Param("url") String url,
- @Param("tableName") String tableName);
-
- @Select("select * from ${table} where id > #{startId} AND id <= #{endId}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("endId") int endId);
-
- @Insert("insert into "
- + "${table}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{detailHtml.html},#{detailHtml.url},#{detailHtml.crawledTime},#{detailHtml.history},#{detailHtml.pageMd5},#{detailHtml.urlMd5})")
- public int insertDetailHtml(@Param("detailHtml")DetailHtml detailHtml, @Param("table") String table);
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorPageDao.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorPageDao.java
deleted file mode 100644
index d3c2ae990..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorPageDao.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import java.util.List;
-
-import net.trustie.webmagic.one.model.AbstractHtml;
-import net.trustie.webmagic.one.model.URL;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-
-public interface ErrorPageDao {
-
- @Insert("insert into "
- + "${table}"
- + " (`html`,`url`,`crawledTime`,`pageMd5`,`urlMd5`,`type`) values (#{html.html},#{html.url},#{html.crawledTime},#{html.pageMd5},#{html.urlMd5},#{html.type})")
- public int insertPage(@Param("table") String table, @Param("html")AbstractHtml html);
-
- @Select("select * from ${table} where id > #{startId} LIMIT #{batchSize}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("batchSize") int batchSize);
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorURLDAO.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorURLDAO.java
deleted file mode 100644
index 0c989c419..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorURLDAO.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import net.trustie.webmagic.one.model.URL;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-
-public interface ErrorURLDAO {
- @Insert("insert ${table} (`id`,`url`) values (#{id},#{url})")
- public int insertURL(@Param("table") String table, URL url);
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ListHtmlDao.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ListHtmlDao.java
deleted file mode 100644
index 552da0db6..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ListHtmlDao.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-import java.util.List;
-
-/**
- * Created by ganyiang on 2014/12/19.
- */
-public interface ListHtmlDao {
- @Insert("insert into "
- + "${table}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{listHtml.html},#{listHtml.url},#{listHtml.crawledTime},#{listHtml.history},#{listHtml.pageMd5},#{listHtml.urlMd5})")
- public int insertListHtml(@Param("listHtml")ListHtml listHtml, @Param("table") String table);
-
- @Select("select html from ${listTableName} where extracted is null limit #{startLine},#{endLine}")
- public List getListHtml(@Param("listTableName") String tableName,
- @Param("startLine") int startLine, @Param("endLine") int endLine);
-
- @Insert("insert ${postUrlsTableName} (`url`,`extractedTime`) values (#{urlPost},#{extractedTime})")
- public int addPostUrls(@Param("postUrlsTableName") String tableName,
- @Param("urlPost") String urlPost,
- @Param("extractedTime") String extractedTime);
-
- @Update("update ${listTableName} set extracted=#{extracted} where pageMd5=#{pageMd5}")
- public int timestamp(@Param("listTableName") String listTableName,
- @Param("pageMd5") String pageMd5,
- @Param("extracted") String extracted);
-
- @Select("select * from ${table} where id > #{startId} LIMIT #{batchSize}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("batchSize") int batchSize);
-
- @Update("create table ${table} ${fields}")
- public int createTable(@Param("table") String table, @Param("fields") String fields);
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/PointerDAO.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/PointerDAO.java
deleted file mode 100644
index 49d14af05..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/PointerDAO.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-import org.apache.ibatis.annotations.Param;
-
-public interface PointerDAO {
-
- @Select("select pointer from ${table} where table_name = #{table_name}")
- public int readPointer(@Param("table")String table,@Param("table_name") String tableName);
-
-
- @Insert("insert into ${table} (`table_name`, `pointer`) values (#{table_name}, #{pointer})")
- public int insertPointer(@Param("table") String table, @Param("table_name") String tableName,
- @Param("pointer") Integer pointer);
-
- @Update("update ${table} set pointer=#{pointer} where table_name=#{table_name}")
- public int updatePointer(@Param("table") String table, @Param("table_name") String tableName,
- @Param("pointer") int pointer);
-
-// @Select("select pointer from pointers where table_name = #{table_name}")
-// public String readMode(@Param("table_name") String table);
-//
-// @Insert("insert into pointers (`table_name`, `pointer`) values (#{table_name}, #{pointer})")
-// public int insertMode(@Param("table_name") String table,
-// @Param("pointer") Integer pointer);
-//
-// @Update("update pointers set pointer=#{pointer} where table_name=#{table_name}")
-// public int updateMode(@Param("table_name") String table,
-// @Param("pointer") int pointer);
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/URLDao.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/URLDao.java
deleted file mode 100644
index c90e2d71a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/URLDao.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import net.trustie.webmagic.one.model.URL;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-import java.util.List;
-
-/**
- * Created by gan on 2014/11/25.
- */
-public interface URLDao {
- @Insert("insert ${listTableName} (`url`,`crawledTime`) values (#{urlPost},#{crawledTime})")
- public int add(@Param("urlPost") String urlPost,
- @Param("crawledTime") String crawledTime,
- @Param("listTableName") String listTableName);
-
- @Select("select url from ${listTableName} where `timestamp` is NULL limit #{startLine},#{endLine}")
- public List getUrl(@Param("listTableName") String listTableName,
- @Param("startLine") int startLine, @Param("endLine") int endLine);
-
- @Update("update ${listTableName} set timestamp=#{timestamp} where url=#{url}")
- public int timestamp(@Param("timestamp") String timestamp,
- @Param("url") String url,
- @Param("listTableName") String listTableName);
-
- @Select("select url from target_url")
- public List getAllUrl();
-
- @Select("select * from ${table} where id > #{startId} LIMIT #{batchSize}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("batchSize") int batchSize);
-
- @Insert("insert ${table} (`url`, `extractedTime`) values (#{url}, #{extractedTime})")
- public int addURL(@Param("table") String table,
- @Param("url") String url, @Param("extractedTime") String timestamp);
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java
deleted file mode 100644
index f78ae15b3..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java
+++ /dev/null
@@ -1,134 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-public abstract class AbstractHtml {
- private int id;
- private String url;
- private String html;
- private String crawledTime;
- private String urlMd5 = "";
- private String pageMd5 = "";
- private int statusCode = 200;
- // 页面类型
- private String type = "Detail";
-
- /**
- * @return the type
- */
- public String getType() {
- return type;
- }
-
- /**
- * @param type
- * the type to set
- */
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- * @return the id
- */
- public int getId() {
- return id;
- }
-
- /**
- * @return the url
- */
- public String getUrl() {
- return url;
- }
-
- /**
- * @return the html
- */
- public String getHtml() {
- return html;
- }
-
- /**
- * @return the crawledTime
- */
- public String getCrawledTime() {
- return crawledTime;
- }
-
- /**
- * @return the urlMd5
- */
- public String getUrlMd5() {
- return urlMd5;
- }
-
- /**
- * @return the pageMd5
- */
- public String getPageMd5() {
- return pageMd5;
- }
-
- /**
- * @return the statusCode
- */
- public int getStatusCode() {
- return statusCode;
- }
-
- /**
- * @param id
- * the id to set
- */
- public void setId(int id) {
- this.id = id;
- }
-
- /**
- * @param url
- * the url to set
- */
- public void setUrl(String url) {
- this.url = url;
- }
-
- /**
- * @param html
- * the html to set
- */
- public void setHtml(String html) {
- this.html = html;
- }
-
- /**
- * @param crawledTime
- * the crawledTime to set
- */
- public void setCrawledTime(String crawledTime) {
- this.crawledTime = crawledTime;
- }
-
- /**
- * @param urlMd5
- * the urlMd5 to set
- */
- public void setUrlMd5(String urlMd5) {
- this.urlMd5 = urlMd5;
- }
-
- /**
- * @param pageMd5
- * the pageMd5 to set
- */
- public void setPageMd5(String pageMd5) {
- this.pageMd5 = pageMd5;
- }
-
- /**
- * @param statusCode
- * the statusCode to set
- */
- public void setStatusCode(int statusCode) {
- this.statusCode = statusCode;
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java
deleted file mode 100644
index aaaf5caf2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-/**
- * Created by Administrator on 2014/11/16.
- */
-public class DetailHtml extends AbstractHtml{
- private int history = 0;
-
- /**
- *
- */
- public DetailHtml() {
- }
-
- /**
- * @return the history
- */
- public int getHistory() {
- return history;
- }
-
- /**
- * @param history the history to set
- */
- public void setHistory(int history) {
- this.history = history;
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/ListHtml.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/ListHtml.java
deleted file mode 100644
index e582da0ff..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/ListHtml.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-import com.sun.jna.platform.win32.Sspi.TimeStamp;
-
-/**
- * Created by Administrator on 2014/12/21.
- */
-public class ListHtml extends AbstractHtml{
- private int history;
- private String extracted;
- /**
- *
- */
- public ListHtml() {
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/URL.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/URL.java
deleted file mode 100644
index de1831d53..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/URL.java
+++ /dev/null
@@ -1,92 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-import java.sql.Timestamp;
-
-
-/**
- * Created by Administrator on 2014/11/27.
- */
-
-public class URL {
- private int id;
- private String url;
- private String type="Detail";
- //private String created
-
- /**
- * @param id
- * @param url
- * @param createdAt
- */
- public URL(int id, String url, Timestamp createdAt) {
- this.id = id;
- this.url = url;
- this.createdAt = createdAt;
- }
-
- /**
- * @return the type
- */
- public String getType() {
- return type;
- }
-
- /**
- * @param type the type to set
- */
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- *
- */
- public URL() {
- }
-
- private Timestamp createdAt;
-
- /**
- * @return the id
- */
- public int getId() {
- return id;
- }
-
- /**
- * @return the url
- */
- public String getUrl() {
- return url;
- }
-
- /**
- * @return the createdAt
- */
- public Timestamp getCreatedAt() {
- return createdAt;
- }
-
- /**
- * @param id the id to set
- */
- public void setId(int id) {
- this.id = id;
- }
-
- /**
- * @param url the url to set
- */
- public void setUrl(String url) {
- this.url = url;
- }
-
- /**
- * @param createdAt the createdAt to set
- */
- public void setCreatedAt(Timestamp createdAt) {
- this.createdAt = createdAt;
- }
-
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/ConsolePipelineOther.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/ConsolePipelineOther.java
deleted file mode 100644
index 1ea94e861..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/ConsolePipelineOther.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class ConsolePipelineOther implements Pipeline {
- @Override
- public void process(ResultItems resultItems, Task task) {
- System.out.println("get page: " + resultItems.getRequest().getUrl());
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter.java
deleted file mode 100644
index 50262b996..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.handler.PatternRequestMatcher;
-import us.codecraft.webmagic.handler.RequestMatcher;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class HtmlSubProcessorPatter extends PatternRequestMatcher implements
- SubPageProcessor {
-
- /**
- * @param pattern
- * url pattern to handle
- */
- public HtmlSubProcessorPatter(String pattern) {
- super(pattern);
- }
-
- @Override
- public MatchOther processPage(Page page) {
- System.out.println("i konwn it!URL_LIST");
- return MatchOther.YES;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter2.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter2.java
deleted file mode 100644
index 81e5f300b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter2.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.handler.PatternProcessor;
-import us.codecraft.webmagic.handler.PatternRequestMatcher;
-import us.codecraft.webmagic.handler.RequestMatcher;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-class HtmlSubProcessorPatter2 extends PatternProcessor {
-
- /**
- * @param pattern
- * url pattern to handle
- */
- public HtmlSubProcessorPatter2(String pattern) {
- super(pattern);
- }
-
- @Override
- public MatchOther processPage(Page page) {
- return null;
- }
-
- @Override
- public MatchOther processResult(ResultItems resultItems, Task task) {
- return null;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorTest2.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorTest2.java
deleted file mode 100644
index 08676cb50..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorTest2.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.handler.PatternRequestMatcher;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class HtmlSubProcessorTest2 extends PatternRequestMatcher implements
- SubPageProcessor {
-
- /**
- * @param pattern
- * url pattern to handle
- */
- public HtmlSubProcessorTest2(String pattern) {
- super(pattern);
- }
-
- @Override
- public MatchOther processPage(Page page) {
- System.out.println("i got it!URL_POST");
- return MatchOther.YES;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate.java
deleted file mode 100644
index 03461306f..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.model.OOSpider;
-import us.codecraft.webmagic.pipeline.PageModelPipeline;
-
-/**
- * Created by gyiang on 2014/8/19.
- */
-public class AutoUpdate {
-
- // private static Spider s;
- public static void go(Spider spider, int startIndex, int endIndex, int f,
- String purl, String furl, int sleepTime, int threadNum,
- PageModelPipeline pageModelPipeline, Class... pageModels) {
- // StoreConf.StoreXML(spider,threadNum,startIndex,endIndex,f,sleepTime);
- OrderSpider.startSpider(spider, startIndex, endIndex, f, purl, furl,0);
-
- while (true) {
- LoadConf prop = new LoadConf(spider.getSite().getDomain());
- Spider s = OOSpider
- .create(Site.me().setRetryTimes(prop.getRetryTimes())
- .setCycleRetryTimes(prop.getCycleRetryTimes())
- .setSleepTime(prop.getSleepTimeThread())
- .setUserAgent(prop.getUserAgent())
- .setDomain(prop.getDomain()), pageModelPipeline,
- pageModels).setSpawnUrl(prop.isSpawnUrl())
- // .setScheduler(spider.getScheduler())
- .thread(threadNum);
- OrderSpider.startSpider(s, prop.getStartPageIndex(),
- prop.getEndInPageIndex(), prop.getPageF(), purl, furl,0);
- // s= spider;
- try {
- Thread.sleep(prop.getSleepTimeThread());
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate_.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate_.java
deleted file mode 100644
index ad222eeeb..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate_.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.model.OOSpider;
-import us.codecraft.webmagic.pipeline.PageModelPipeline;
-
-/**
- * Created by gyiang on 2014/8/19.
- */
-public class AutoUpdate_ {
-
- // private static Spider s;
- public static void go(Spider spider, int startIndex, int endIndex, int f,
- String purl, String furl, int sleepTime, int threadNum,
- PageModelPipeline pageModelPipeline, Class... pageModels) {
- OrderSpider.startSpider(spider, startIndex, endIndex, f, purl, furl,0);
- while (true) {
- Spider s = OOSpider
- .create(Site
- .me()
- .setRetryTimes(spider.getSite().getRetryTimes())
- .setCycleRetryTimes(
- spider.getSite().getCycleRetryTimes())
- .setSleepTime(spider.getSite().getSleepTime())
- .setUserAgent(spider.getSite().getUserAgent()),
- pageModelPipeline, pageModels)
- .setSpawnUrl(spider.isSpawnUrl())
- // .setScheduler(spider.getScheduler())
- .thread(threadNum);
- OrderSpider.startSpider(s, startIndex, endIndex, f, purl, furl,0);
- // s= spider;
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DBPipeline.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DBPipeline.java
deleted file mode 100644
index 821fd3715..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DBPipeline.java
+++ /dev/null
@@ -1,60 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-public abstract class DBPipeline implements Pipeline {
- private String listHtmlTableName = "";
- private String detailHtmlTableName = "";
- private String errorPageTableName = "";
- private String errorUrlTableName = "";
- /**
- * @return the listHtmlTableName
- */
- public String getListHtmlTableName() {
- return listHtmlTableName;
- }
- /**
- * @return the detailHtmlTableName
- */
- public String getDetailHtmlTableName() {
- return detailHtmlTableName;
- }
- /**
- * @return the errorPageTableName
- */
- public String getErrorPageTableName() {
- return errorPageTableName;
- }
- /**
- * @return the errorUrlTableName
- */
- public String getErrorUrlTableName() {
- return errorUrlTableName;
- }
- /**
- * @param listHtmlTableName the listHtmlTableName to set
- */
- public void setListHtmlTableName(String listHtmlTableName) {
- this.listHtmlTableName = listHtmlTableName;
- }
- /**
- * @param detailHtmlTableName the detailHtmlTableName to set
- */
- public void setDetailHtmlTableName(String detailHtmlTableName) {
- this.detailHtmlTableName = detailHtmlTableName;
- }
- /**
- * @param errorPageTableName the errorPageTableName to set
- */
- public void setErrorPageTableName(String errorPageTableName) {
- this.errorPageTableName = errorPageTableName;
- }
- /**
- * @param errorUrlTableName the errorUrlTableName to set
- */
- public void setErrorUrlTableName(String errorUrlTableName) {
- this.errorUrlTableName = errorUrlTableName;
- }
-
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DateFormat.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DateFormat.java
deleted file mode 100644
index f9e5c02c6..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DateFormat.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.sql.Timestamp;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Locale;
-
-/**
- * Created by Administrator on 2014/10/2.
- */
-public class DateFormat {
-
- public static String formatFrom(String formatString, String format) {
-
- SimpleDateFormat bartDateFormat = new SimpleDateFormat(format,
- Locale.ENGLISH);
- SimpleDateFormat bartDateFormat2 = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
-
- String date = "2000-01-01 00:00:00";
- try {
- date = bartDateFormat2.format(bartDateFormat.parse(formatString))
- .toString();
- } catch (ParseException e) {
- e.printStackTrace();
- }
- return date;
- }
-
- public static String formatFrom2(String strTime) {
-
- if (strTime.contains("昨天"))
- strTime = "1天前";
- String time = "2000-01-01 00:00:00";
- int vanishTime = Integer.parseInt(strTime.replaceAll("[^\\d]", ""));
- String[] ways = strTime.split("\\d+");
-
- String unit = ways[1];
- if (unit.contains("秒前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.SECOND)).toString();
- } else if (unit.contains("分钟前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.MINUTE)).toString();
- } else if (unit.contains("小时前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.HOUR)).toString();
- } else if (unit.contains("天前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.DAY)).toString();
- } else if (unit.contains("个月前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.MONTH)).toString();
- } else if (unit.contains("年前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.YEAR)).toString();
- } else {
- return null;
- }
- return time;
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/FileReader.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/FileReader.java
deleted file mode 100644
index 6d6baa65e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/FileReader.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.util.Scanner;
-
-public class FileReader {
- public static String readFile(String path) {
- File file = new File(path);
- String rt = "";
- try {
- Scanner sc = new Scanner(file);
- while(sc.hasNextLine()){
- rt += sc.nextLine();
- rt+=" ";
- }
- } catch (FileNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return rt;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetJSON.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetJSON.java
deleted file mode 100644
index ed4c753b6..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetJSON.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.protocol.HTTP;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-/**
- * Created by Administrator on 2014/7/31.
- */
-public class GetJSON {
- public static void main(String[] args) throws JSONException, IOException {
-
- JSONObject json = new JSONObject();
- json.put("period", "7");
- json.put(
- "__RequestVerificationToken",
- "G7lAAjIljV3yoksJNBfesXVbBm9M4ZF0nS8yjs0i2QYkiay4O7ybpgVD07NOiXhUkCURkwW0wLdAIiOagn-nAoJJxjs0I-oYRyMXX8Z6pixZ7DNRWhrfd49RTTmEPV8APA0HDg2");
- HttpPost httpPost = new HttpPost(
- "http://kinectmultipoint.codeplex.com/stats/getActivity");
- StringEntity entity = new StringEntity(json.toString(), HTTP.UTF_8);
- entity.setContentType("application/json");
- httpPost.setEntity(entity);
- // HttpClient client = new DefaultHttpClient();
- HttpClient client = null;
- HttpResponse response = client.execute(httpPost);
- System.out.println(response.toString());
-
- /*
- * List params = new ArrayList();
- * params.add(new BasicNameValuePair("period", "7")); params.add(new
- * BasicNameValuePair("__RequestVerificationToke...",
- * "G7lAAjIljV3yoksJNBfesXVbBm9M4ZF0nS8yjs0i2QYkiay4O7ybpgVD07NOiXhUkCURkwW0wLdAIiOagn-nAoJJxjs0I-oYRyMXX8Z6pixZ7DNRWhrfd49RTTmEPV8APA0HDg2"
- * )); //要传递的参数. String url =
- * "http://kinectmultipoint.codeplex.com/stats/getActivity?" +
- * URLEncodedUtils.format(params, HTTP.UTF_8); //拼接路径字符串将参数包含进去 json =
- * get(url);
- */
- System.out.println(json.get("Visits"));
-
- }
-
- public static JSONObject get(String url) {
-
- // HttpClient client = new DefaultHttpClient();
- HttpClient client = null;
- HttpGet get = new HttpGet(url);
- JSONObject json = null;
- try {
- HttpResponse res = client.execute(get);
- if (res.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- HttpEntity entity = res.getEntity();
- json = new JSONObject(new JSONTokener(new InputStreamReader(
- entity.getContent(), HTTP.UTF_8)));
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
-
- } finally {
- // 关闭连接 ,释放资源
- client.getConnectionManager().shutdown();
- }
- return json;
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetUrlsProjSF.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetUrlsProjSF.java
deleted file mode 100644
index a113a57f7..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetUrlsProjSF.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import org.apache.log4j.Logger;
-
-import java.sql.*;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Created by Administrator on 2014/9/30.
- */
-public class GetUrlsProjSF {
- public static List get() throws SQLException,
- ClassNotFoundException, IllegalAccessException,
- InstantiationException {
- Logger log4j = Logger.getLogger(GetUrlsProjSF.class);
- List l = new ArrayList();
- String sql2 = "select url from proj_sourceforge_url";
- Class.forName("com.mysql.jdbc.Driver").newInstance();
- Connection conn = DriverManager.getConnection(
- "jdbc:mysql://192.168.80.104:3306/oss_tree_fix", "influx",
- "influx1234");
- // Statement stmt = conn.createStatement();
- PreparedStatement ps = conn.prepareStatement(sql2);
- ResultSet rs = ps.executeQuery();
- int i = 0;
- while (rs.next()) {
- String t = rs.getString("url");
- log4j.info("added!#" + t);
- i++;
- l.add(t);
- /*
- * if (i==100) break;
- */
- }
- log4j.info("total urls:" + i);
- log4j.error("well down");
- return l;
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LoadConf.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LoadConf.java
deleted file mode 100644
index 6a9429516..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LoadConf.java
+++ /dev/null
@@ -1,189 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import org.apache.log4j.Logger;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.InvalidPropertiesFormatException;
-import java.util.Properties;
-
-public class LoadConf {
- private int sleepTimeThread = 5000;
- private int retryTimes = 0;
- private int cycleRetryTimes = 0;
- private int timeOut = 5000;
- private String userAgent = "UserAgent.Browser";
- private String charset;
- private int threadNum = 5;
- private boolean isSpawnUrl = true;
- private int startPageIndex = 1;
- private int endInPageIndex = 6;
- private int pageF = 1;
- private int sleepTimeSpider = 36000;
- private String domain;
- private Properties prop = new Properties();
- Logger log4j = Logger.getLogger(LoadConf.class);
-
- public LoadConf() {
- try {
- FileInputStream fis = null;
- fis = new FileInputStream("spider.conf.xml");
- prop.loadFromXML(fis);
- prop.list(System.out);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
-
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- public LoadConf(String confPath) {
- System.out.println(new File(".").getAbsolutePath());
- confPath = "conf/" + "conf." + confPath + ".xml";
- System.err.println(new File(confPath).getAbsolutePath());
- try {
- FileInputStream fis = null;
- fis = new FileInputStream(confPath);
- prop.loadFromXML(fis);
- prop.list(System.out);
- } catch (FileNotFoundException e) {
- log4j.error("FileNotFoundException & loading default config xml! & no default config file");
- e.printStackTrace();
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- private void config() {
- this.sleepTimeThread = Integer.parseInt(prop
- .getProperty("sleepTimeThread"));
- this.timeOut = Integer.parseInt(prop.getProperty("timeOut"));
- this.retryTimes = Integer.parseInt(prop.getProperty("retryTimes"));
- this.cycleRetryTimes = Integer.parseInt(prop
- .getProperty("cycleRetryTimes"));
- this.threadNum = Integer.parseInt(prop.getProperty("threadNum"));
- this.startPageIndex = Integer.parseInt(prop
- .getProperty("startPageIndex"));
- this.endInPageIndex = Integer.parseInt(prop
- .getProperty("endInPageIndex"));
- this.pageF = Integer.parseInt(prop.getProperty("pageF"));
- this.sleepTimeSpider = Integer.parseInt(prop
- .getProperty("sleepTimeSpider"));
- this.isSpawnUrl = Boolean.parseBoolean(prop.getProperty("isSpawnUrl"));
- this.userAgent = prop.getProperty("userAgent");
- this.domain = prop.getProperty("domain");
- }
-
- public int getSleepTimeThread() {
- return sleepTimeThread;
- }
-
- public void setSleepTimeThread(int sleepTimeThread) {
- this.sleepTimeThread = sleepTimeThread;
- }
-
- public int getRetryTimes() {
- return retryTimes;
- }
-
- public void setRetryTimes(int retryTimes) {
- this.retryTimes = retryTimes;
- }
-
- public int getCycleRetryTimes() {
- return cycleRetryTimes;
- }
-
- public void setCycleRetryTimes(int cycleRetryTimes) {
- this.cycleRetryTimes = cycleRetryTimes;
- }
-
- public int getTimeOut() {
- return timeOut;
- }
-
- public void setTimeOut(int timeOut) {
- this.timeOut = timeOut;
- }
-
- public String getUserAgent() {
- return userAgent;
- }
-
- public void setUserAgent(String userAgent) {
- this.userAgent = userAgent;
- }
-
- public String getCharset() {
- return charset;
- }
-
- public void setCharset(String charset) {
- this.charset = charset;
- }
-
- public int getThreadNum() {
- return threadNum;
- }
-
- public void setThreadNum(int threadNum) {
- this.threadNum = threadNum;
- }
-
- public boolean isSpawnUrl() {
- return isSpawnUrl;
- }
-
- public void setSpawnUrl(boolean isSpawnUrl) {
- this.isSpawnUrl = isSpawnUrl;
- }
-
- public int getStartPageIndex() {
- return startPageIndex;
- }
-
- public void setStartPageIndex(int startPageIndex) {
- this.startPageIndex = startPageIndex;
- }
-
- public int getEndInPageIndex() {
- return endInPageIndex;
- }
-
- public void setEndInPageIndex(int endInPageIndex) {
- this.endInPageIndex = endInPageIndex;
- }
-
- public int getPageF() {
- return pageF;
- }
-
- public void setPageF(int pageF) {
- this.pageF = pageF;
- }
-
- public int getSleepTimeSpider() {
- return sleepTimeSpider;
- }
-
- public void setSleepTimeSpider(int sleepTimeSpider) {
- this.sleepTimeSpider = sleepTimeSpider;
- }
-
- public String getDomain() {
- return domain;
- }
-
- public void setDomain(String domain) {
- this.domain = domain;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LongSleepHelper.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LongSleepHelper.java
deleted file mode 100644
index e1f57f422..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LongSleepHelper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import net.trustie.webmagic.one.Configure;
-
-/*
- * 此类用于设置随机长时间段睡眠
- */
-public class LongSleepHelper {
- private long lastSleep;//上一次长睡眠的结束时间
- private int sleepInterval;//下一次长睡眠间隔期
- private int longSleepTime;//此次长睡眠的时间
-
- private int minlongSleepInterval;
- private int maxlongSleepInterval;
- private int minLongSleepTime;
- private int maxLongSleepTime;
- public LongSleepHelper(Configure conf){
- this.minlongSleepInterval = conf.getMinlongSleepInterval();
- this.maxlongSleepInterval = conf.getMaxlongSleepInterval();
- this.minLongSleepTime = conf.getMinlongSleepTime();
- this.maxLongSleepTime = conf.getMaxlongSleepTime();
- }
- //记录当前时间
- public void startMonitor(){
- lastSleep = System.currentTimeMillis();
- //sleepInterval = MyRandomer.getRandomNum(1*60*60*1000,2*60*60*1000);
- sleepInterval = MyRandomer.getRandomNum(this.minlongSleepInterval,this.maxlongSleepInterval);
- }
- public boolean timeToLongSleep(){
- long now = System.currentTimeMillis();
- if(now >= (lastSleep + sleepInterval)){
- //longSleepTime = MyRandomer.getRandomNum(30*60*1000,2*60*60*1000);
- longSleepTime = MyRandomer.getRandomNum(this.minLongSleepTime,this.maxLongSleepTime);
- lastSleep = now + longSleepTime;
- //sleepInterval = MyRandomer.getRandomNum(1*60*60*1000,2*60*60*1000);
- sleepInterval = MyRandomer.getRandomNum(this.minlongSleepInterval,this.maxlongSleepInterval);
- return true;
- }else
- return false;
- }
-
- public int longsleepTime(){
- return longSleepTime;
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/MyRandomer.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/MyRandomer.java
deleted file mode 100644
index 2339716b6..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/MyRandomer.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.util.Random;
-
-
-//用以产生随机数
-public class MyRandomer {
- private static Random random = new Random(66);
-
- //在[min,max)之间产生一个随机数
- public static int getRandomNum(int min,int max){
- return random.nextInt(max-min)+min;
- }
- //在[0,max)之间产生一个随机数
- public static int getRandomNum(int max){
- return random.nextInt(max);
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/OrderSpider.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/OrderSpider.java
deleted file mode 100644
index d5859600d..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/OrderSpider.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.monitor.SpiderMonitor;
-
-import javax.management.JMException;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-import net.trustie.webmagic.one.ListHtmlPipeline;
-
-import org.apache.log4j.Logger;
-
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * Created by gyiang on 2014/8/5.
- */
-public class OrderSpider {
- private static Logger logger = Logger.getLogger(OrderSpider.class);
-
- public static List orderUrl(String purl, String surl, int per,
- int incIndex, int startIndex) {
- List urls = new ArrayList();
- for (int g = startIndex; g < startIndex + per; g++) {
- int pageindex = g + incIndex;
- String url = purl + pageindex + surl;
- urls.add(url);
- }
- return urls;
- }
-
- /*
- * @startIndex起始页
- *
- * @endIndex结束页
- *
- * @per每次生成几个链接
- *
- * @purl 前缀
- *
- * @furl 后缀(p、f是什么意思?) 调用方式: OrderSpider.startSpider(spider, 1, 4,2 ,
- * "http://www.oschina.net/project/list?prefix=&sort=view&p=", "");
- */
- public static void startSpider(Spider spider, int startIndex, int endIndex,
- int per, String purl, String furl, int sleepTimeSpider) {
- for (int i = startIndex; i <= endIndex; i += per) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- spider.startUrls(orderUrl(purl, furl, per, 0, i));
-
- // 爬虫监控用,不需要
- try {
- MBeanServer mbeanServer = ManagementFactory
- .getPlatformMBeanServer();
- ObjectName objName = new ObjectName("WebMagic" + ":name="
- + spider.getSite().getDomain());
- if (mbeanServer.isRegistered(objName))
- mbeanServer.unregisterMBean(objName);
- spider.setUUID(spider.getSite().getDomain());
- SpiderMonitor.instance().register(spider);
- } catch (JMException e) {
- e.printStackTrace();
- }
- spider.run();
- //sleep
- try {
- logger.info("sleep ");
- Thread.sleep(sleepTimeSpider);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Protector.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Protector.java
deleted file mode 100644
index 71ac16b72..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Protector.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import net.trustie.webmagic.one.Configure;
-import us.codecraft.webmagic.Site;
-
-//此类用于设置加强爬虫性能
-public class Protector {
-
- //为了防止ip被封,对设置的参数进行随机化
- public static void setRandomArgs(Site site,Configure conf){
- //随机设置UA
- site.setUserAgent(UserAgentPool.getUserAgent());
- //随机设置每次请求的间隔
- int time = MyRandomer.getRandomNum(conf.getMinInterval(),
- conf.getMaxInterval());
- conf.setSleepTimeSpider(time);
- //site.setSleepTime(time);
- }
-
- public void main(String args[]){
- while(true){
-
- }
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SimHash.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SimHash.java
deleted file mode 100644
index 0fb849f5a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SimHash.java
+++ /dev/null
@@ -1,197 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.StringTokenizer;
-
-/**
- * Created by Administrator on 2014/11/12.
- */
-
-public class SimHash {
-
- private String tokens;
-
- private BigInteger intSimHash;
-
- private String strSimHash;
-
- private int hashbits = 64;
-
- public SimHash(String tokens) {
- this.tokens = tokens;
- this.intSimHash = this.simHash();
- }
-
- public SimHash(String tokens, int hashbits) {
- this.tokens = tokens;
- this.hashbits = hashbits;
- this.intSimHash = this.simHash();
- }
-
- public BigInteger simHash() {
- int[] v = new int[this.hashbits];
- StringTokenizer stringTokens = new StringTokenizer(this.tokens);
- while (stringTokens.hasMoreTokens()) {
- String temp = stringTokens.nextToken();
- BigInteger t = this.hash(temp);
- for (int i = 0; i < this.hashbits; i++) {
- BigInteger bitmask = new BigInteger("1").shiftLeft(i);
- if (t.and(bitmask).signum() != 0) {
- v[i] += 1;
- } else {
- v[i] -= 1;
- }
- }
- }
- BigInteger fingerprint = new BigInteger("0");
- StringBuffer simHashBuffer = new StringBuffer();
- for (int i = 0; i < this.hashbits; i++) {
- if (v[i] >= 0) {
- fingerprint = fingerprint.add(new BigInteger("1").shiftLeft(i));
- simHashBuffer.append("1");
- } else {
- simHashBuffer.append("0");
- }
- }
- this.strSimHash = simHashBuffer.toString();
- System.out.println(this.strSimHash + " length "
- + this.strSimHash.length());
- return fingerprint;
- }
-
- private BigInteger hash(String source) {
- if (source == null || source.length() == 0) {
- return new BigInteger("0");
- } else {
- char[] sourceArray = source.toCharArray();
- BigInteger x = BigInteger.valueOf(((long) sourceArray[0]) << 7);
- BigInteger m = new BigInteger("1000003");
- BigInteger mask = new BigInteger("2").pow(this.hashbits).subtract(
- new BigInteger("1"));
- for (char item : sourceArray) {
- BigInteger temp = BigInteger.valueOf((long) item);
- x = x.multiply(m).xor(temp).and(mask);
- }
- x = x.xor(new BigInteger(String.valueOf(source.length())));
- if (x.equals(new BigInteger("-1"))) {
- x = new BigInteger("-2");
- }
- return x;
- }
- }
-
- /**
- * 取两个二进制的异或,统计为1的个数,就是海明距离
- *
- * @param other
- * @return
- */
-
- public int hammingDistance(SimHash other) {
-
- BigInteger x = this.intSimHash.xor(other.intSimHash);
- int tot = 0;
-
- // 统计x中二进制位数为1的个数
- // 我们想想,一个二进制数减去1,那么,从最后那个1(包括那个1)后面的数字全都反了,对吧,然后,n&(n-1)就相当于把后面的数字清0,
- // 我们看n能做多少次这样的操作就OK了。
-
- while (x.signum() != 0) {
- tot += 1;
- x = x.and(x.subtract(new BigInteger("1")));
- }
- return tot;
- }
-
- /**
- * calculate Hamming Distance between two strings 二进制怕有错,当成字符串,作一个,比较下结果
- *
- * @author
- * @param str1
- * the 1st string
- * @param str2
- * the 2nd string
- * @return Hamming Distance between str1 and str2
- */
- public int getDistance(String str1, String str2) {
- int distance;
- if (str1.length() != str2.length()) {
- distance = -1;
- } else {
- distance = 0;
- for (int i = 0; i < str1.length(); i++) {
- if (str1.charAt(i) != str2.charAt(i)) {
- distance++;
- }
- }
- }
- return distance;
- }
-
- /**
- * 如果海明距离取3,则分成四块,并得到每一块的bigInteger值 ,作为索引值使用
- *
- * @param simHash
- * @param distance
- * @return
- */
- public List subByDistance(SimHash simHash, int distance) {
- int numEach = this.hashbits / (distance + 1);
- List characters = new ArrayList();
-
- StringBuffer buffer = new StringBuffer();
-
- int k = 0;
- for (int i = 0; i < this.intSimHash.bitLength(); i++) {
- boolean sr = simHash.intSimHash.testBit(i);
-
- if (sr) {
- buffer.append("1");
- } else {
- buffer.append("0");
- }
-
- if ((i + 1) % numEach == 0) {
- BigInteger eachValue = new BigInteger(buffer.toString(), 2);
- System.out.println("----" + eachValue);
- buffer.delete(0, buffer.length());
- characters.add(eachValue);
- }
- }
-
- return characters;
- }
-
- public static void main(String[] args) {
- String s = "This is a test string for testing";
-
- SimHash hash1 = new SimHash(s, 64);
- System.out.println(hash1.intSimHash + " "
- + hash1.intSimHash.bitLength());
-
- hash1.subByDistance(hash1, 3);
-
- System.out.println("\n");
- s = "This is a test string for testing, This is a test string for testing abcdef";
- SimHash hash2 = new SimHash(s, 64);
- System.out.println(hash2.intSimHash + " "
- + hash2.intSimHash.bitCount());
- hash1.subByDistance(hash2, 3);
- s = "This is a test string for testing als";
- SimHash hash3 = new SimHash(s, 64);
- System.out.println(hash3.intSimHash + " "
- + hash3.intSimHash.bitCount());
- hash1.subByDistance(hash3, 3);
- System.out.println("============================");
- int dis = hash1.getDistance(hash1.strSimHash, hash2.strSimHash);
-
- System.out.println(hash1.hammingDistance(hash2) + " " + dis);
-
- int dis2 = hash1.getDistance(hash1.strSimHash, hash3.strSimHash);
-
- System.out.println(hash1.hammingDistance(hash3) + " " + dis2);
-
- }
-}
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StoreConf.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StoreConf.java
deleted file mode 100644
index 63dfddf7b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StoreConf.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Spider;
-
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Properties;
-
-public class StoreConf {
-
- public static void StoreXML(Spider spider, int threadNum, int startIndex,
- int endIndex, int pageF, int sleepTimeSpider) {
- Properties prop = new Properties();
- prop.setProperty("domain", spider.getSite().getDomain() + "");
- prop.setProperty("retryTimes", spider.getSite().getRetryTimes() + "");
- prop.setProperty("cycleRetryTimes", spider.getSite()
- .getCycleRetryTimes() + "");
- prop.setProperty("timeOut", spider.getSite().getTimeOut() + "");
- prop.setProperty("sleepTimeThread", spider.getSite().getSleepTime()
- + "");
- prop.setProperty("userAgent", spider.getSite().getUserAgent() + "");
- prop.setProperty("threadNum", threadNum + "");
- prop.setProperty("isSpawnUrl", spider.isSpawnUrl() + "");
- prop.setProperty("startPageIndex", startIndex + "");
- prop.setProperty("endInPageIndex", endIndex + "");
- prop.setProperty("pageF", pageF + "");
- prop.setProperty("sleepTimeSpider", sleepTimeSpider + "");
- prop.setProperty("proxy", spider.getSite().getHttpProxyPool() + "");
- FileOutputStream fos;
- try {
- fos = new FileOutputStream("spider.conf.xml");
- prop.storeToXML(fos, "Spider");
- fos.flush();
- fos.close();
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StringUtils.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StringUtils.java
deleted file mode 100644
index d02f538b2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StringUtils.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Created by Administrator on 2014/7/22.
- */
-public class StringUtils {
-
- public static List getDigit(String text) {
- List digitList = new ArrayList();
- Pattern p = Pattern.compile("(\\d+)");
- Matcher m = p.matcher(text);
- while (m.find()) {
- String find = m.group(1).toString();
- digitList.add(Long.valueOf(find));
- }
- return digitList;
- }
-
- public static String getNumber(String text) {
- Pattern p = Pattern.compile("(\\d+)");
- Matcher m = p.matcher(text);
- if (m.find())
- return m.group(1).toString();
- else
- return "0";
- }
-
- public static Set removeDuplicate(List strs) {
- Set rt = new HashSet();
- for (String str : strs) {
- rt.add(str);
- }
- return rt;
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SubSite.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SubSite.java
deleted file mode 100644
index 93a76fd26..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SubSite.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package net.trustie.webmagic.utils;
-
-public class SubSite {
- String name;
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public int getStartPageIndex() {
- return startPageIndex;
- }
- public void setStartPageIndex(int startPageIndex) {
- this.startPageIndex = startPageIndex;
- }
- public int getEndInPageIndex() {
- return endInPageIndex;
- }
- public void setEndInPageIndex(int endInPageIndex) {
- this.endInPageIndex = endInPageIndex;
- }
- public String getUrlPost() {
- return urlPost;
- }
- public void setUrlPost(String urlPost) {
- this.urlPost = urlPost;
- }
- public String getFixAllRelativeHrefs() {
- return fixAllRelativeHrefs;
- }
- public void setFixAllRelativeHrefs(String fixAllRelativeHrefs) {
- this.fixAllRelativeHrefs = fixAllRelativeHrefs;
- }
- public String getPrefixUrl() {
- return prefixUrl;
- }
- public void setPrefixUrl(String prefixUrl) {
- this.prefixUrl = prefixUrl;
- }
- public String getPostfixUrl() {
- return postfixUrl;
- }
- public void setPostfixUrl(String postfixUrl) {
- this.postfixUrl = postfixUrl;
- }
- public int getIncrementalPages() {
- return incrementalPages;
- }
- public void setIncrementalPages(int incrementalPages) {
- this.incrementalPages = incrementalPages;
- }
- public int getIncrementSleepTime() {
- return incrementSleepTime;
- }
- public void setIncrementSleepTime(int incrementSleepTime) {
- this.incrementSleepTime = incrementSleepTime;
- }
- public int getMode() {
- return mode;
- }
- public void setMode(int mode) {
- this.mode = mode;
- }
- int startPageIndex;
- int endInPageIndex;
- String urlPost;
- String fixAllRelativeHrefs;
- String prefixUrl;
- String postfixUrl;
- int incrementalPages;
- int incrementSleepTime;
- int mode;
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/TableHelper.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/TableHelper.java
deleted file mode 100644
index bfc786f09..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/TableHelper.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import javax.annotation.Resource;
-
-import org.springframework.stereotype.Component;
-
-import net.trustie.webmagic.one.dao.CreateTableDAO;
-
-@Component("tableHelper")
-public class TableHelper {
- @Resource
- private CreateTableDAO dao;
-
- public boolean isTableExist(String tableName){
- try{
- dao.isTableExist(tableName);
- //System.out.println(i);
- return true;
- }catch(Exception e){
- //e.printStackTrace();
- return false;
- }
-
- }
-
- public void createTable(String table, String fields){
- dao.createTable(table, fields);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgent.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgent.java
deleted file mode 100644
index b006a4763..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgent.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package net.trustie.webmagic.utils;
-
-/**
- * Created by gyiang on 2014/8/2.
- */
-public class UserAgent {
- public static final String Baidu = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
- public static final String Google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
- public static final String Youdao = "Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )";
- public static final String Sogou = "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)";
- public static final String Soso = "Sosospider+(+http://help.soso.com/webspider.htm)";
- public static final String Yahoo = "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)";
- public static final String Jike = "Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)";
- public static final String Sinaiask = "iaskspider/2.0(+http://iask.com/help/help_index.html\")";
- public static final String Msn = "msnbot/1.0 (+http://search.msn.com/msnbot.htm)";
-
- public static final String PicBaidu = "Baiduspider-image";
- public static final String PicGoogle = "Googlebot-Image/1.0";
- public static final String PicSogou = "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)";
-
- public static final String Browser = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36";
- // public static final String
- // BrowserLinux="Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803";
- // public static final String
- // BrowserWindows="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET CLR 1.1.4322; .NET4.0C)";
-
- public static final String Mobile = "Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 930) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537";
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgentPool.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgentPool.java
deleted file mode 100644
index c20e29274..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgentPool.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.util.Random;
-
-
-
-/*
- * Star Lee, 2015-04-13
- *this class is used to generate a random User Agent
- */
-public class UserAgentPool {
-
- //user agent pool
- private static String[] userAgents = {
-
-
- "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
- "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
- "Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )",
- "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
- "Sosospider+(+http://help.soso.com/webspider.htm)",
- "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",
- "Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)",
- "iaskspider/2.0(+http://iask.com/help/help_index.html\")",
- "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
-
-
- //ie6clients
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; XMPP Tiscali Communicator v.10.0.2; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
-
- //ie7clients
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0 ; .NET CLR 2.0.50215; SL Commerce Client v1.0; Tablet PC 2.0",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506)" ,// Windows Mail on Vista
-
- // ie8clients
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506; Media Center PC 5.0; .NET CLR 1.1.4322)",
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; .NET CLR 2.0.50727; SLCC1; Media Center PC 5.0; .NET CLR 3.0.04506)",
-
- // ie9clients
- "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)",
- "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)",
-
- //ie10clients
- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
- "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
-
- //ie11clients
- "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
- "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; ASU2JS; rv:11.0) like Gecko", //64bit Win8
- //chrome
- "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.309.0 Safari/532.9",
-
- //chrome8
- "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.558.0 Safari/534.10",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML, like Gecko) Ubuntu/10.10 Chrome/8.1.0.0 Safari/540.0",
-
- //chrome9
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML,like Gecko) Chrome/9.1.0.0 Safari/540.0",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.600.0 Safari/534.14",
-
- //chrome10
- "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.613.0 Chrome/10.0.613.0 Safari/534.15",
-
- //chrome11
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.697.0 Safari/534.24",
-
- //chrome12
- "Mozilla/5.0 (X11; CrOS i686 12.0.742.91) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.93 Safari/534.30",
-
- //chrome13
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1",
-
- //chrome14
- "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.815.0 Safari/535.1",
-
- //chrome29
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36",
-
- //chrome32
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36",
-
- //chrome36
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36",
-
- //chrome33
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36",
-
- //chrome31
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36",
-
- //firefox3
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.14) Gecko/2009090216 Ubuntu/9.04 (jaunty) Firefox/3.0.14",
-
- //firefox4
- "Mozilla/5.0 (X11; Linux x86_64; rv:2.0b4) Gecko/20100818 Firefox/4.0b4",
- "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b9pre) Gecko/20101228 Firefox/4.0b9pre",
-
- //firefox5
- "Mozilla/5.0 (Windows NT 6.1; U; ru; rv:5.0.1.6) Gecko/20110501 Firefox/5.0.1 Firefox/5.0.1",
- "Mozilla/5.0 (X11; U; Linux i586; de; rv:5.0) Gecko/20100101 Firefox/5.0",
-
- //firefox6
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110612 Firefox/6.0a2",
-
- //firefox19
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:19.0) Gecko/20100101 Firefox/19.0",
-
- //firefox20
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:20.0) Gecko/20100101 Firefox/20.0",
-
- //firefox25
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0",
-
- //firefox28
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0",
- //safari
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-us) AppleWebKit/525.28.3 (KHTML, like Gecko) Version/3.2.3 Safari/525.28.3",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-gb) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6",
-
- //safari8
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25",
-
- //safari7
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71",
-
- //safari6
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.28.10 (KHTML, like Gecko) Version/6.0.3 Safari/536.28.10",
-
- //safari5
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-us) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
-
- //safari4
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; es-ES) AppleWebKit/531.22.7 (KHTML, like Gecko) Version/4.0.5 Safari/531.22.7",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/4.0.1 Safari/530.18",
-
- //opera
- "Opera/8.0 (Macintosh; PPC Mac OS X; U; en)",
-
- //opera9
- "Opera/9.52 (Windows NT 5.1; U; en)",
- "Opera/9.20 (Macintosh; Intel Mac OS X; U; en)",
-
- //opera10
- "Opera/9.80 (Windows NT 5.2; U; en) Presto/2.2.15 Version/10.10",
- "Opera/9.80 (Macintosh; Intel Mac OS X; U; en) Presto/2.6.30 Version/10.61",
-
- //opera11
- "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
- "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
-
- //opera12
- "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
- "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
-
- //opera15
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36 OPR/15.0.1147.100",
-
- //opera16
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36 OPR/16.0.1196.73",
-
- //opera17
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36 OPR/17.0.1241.53",
-
- //opera18
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.63",
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 OPR/18.0.1284.68",
- "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49",
-
- //opera19
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36 OPR/19.0.1326.39 (Edition Next)",
- "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36 OPR/19.0.1326.56",
- "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36 OPR/19.0.1326.59",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36 OPR/19.0.1326.63",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36 OPR/19.0.1326.63",
-
- //opera20
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.91 Safari/537.36 OPR/20.0.1387.37 (Edition Next)",
- "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.46 Safari/537.36 OPR/20.0.1387.16 (Edition Developer)",
-
- //opera23
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36 OPR/23.0.1522.60",
-
- //opera24
- "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.94 Safari/537.36 OPR/24.0.1558.51 (Edition Next)"
- };
-
-
-
-
- public static String getUserAgent(){
- return userAgents[MyRandomer.getRandomNum(userAgents.length)];
- }
-
- /*public static void main(String args[]){
-
- for(int i = 0; i < 20; i++){
- System.out.println(getUserAgent());
- }
-
- }*/
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Utils.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Utils.java
deleted file mode 100644
index 324e89c7e..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Utils.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import net.trustie.webmagic.one.Configure;
-
-public class Utils {
- public static List getProxies(Configure conf) {
- List rt = new ArrayList();
- String[] proxy = new String[2];
- proxy[0] = conf.getProxyIp();
- proxy[1] = conf.getProxyPort();
- rt.add(proxy);
- return rt;
- }
-
- public static String getNow() {
- SimpleDateFormat crawledTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- Date date = new Date();
- // String timestamp = timestampFormat.format(date);
- return crawledTimeFormat.format(date);
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/VanishTime.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/VanishTime.java
deleted file mode 100644
index cd198de43..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/VanishTime.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package net.trustie.webmagic.utils;
-
-public class VanishTime {
- public final static long SECOND = 1000;
- public final static long MINUTE = 60 * SECOND;
- public final static long HOUR = 60 * MINUTE;
- public final static long DAY = 24 * HOUR;
- public final static long MONTH = 30 * DAY;
- public final static long YEAR = 12 * MONTH;
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/xmlToDB.java b/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/xmlToDB.java
deleted file mode 100644
index c14db3011..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/xmlToDB.java
+++ /dev/null
@@ -1,195 +0,0 @@
-package net.trustie.webmagic.utils;
-
-/**
- * Created by gyiang on 2014/8/1.
- */
-
-/**
- 纯属本人习惯.仅供参考!
- */
-import java.io.File;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Iterator;
-import org.dom4j.Document;
-import org.dom4j.Element;
-import org.dom4j.io.SAXReader;
-
-public class xmlToDB {
-
- /**
- * 给定XML,解析后插入数据库
- *
- * @param filePath
- * 文件的路径名:例如:"F:/creatXml.xml"
- * @throws ClassNotFoundException
- *
- */
-
- public static void importXml(String filePath) {
-
- System.out.println("============开始导入============");
-
- // -------------jdbc代码
- try {
- Class.forName("com.mysql.jdbc.Driver");
- String url = "jdbc:mysql://localhost:3306/influx";
- String user = "root";
- String password = "a1passz";
- Connection con = DriverManager.getConnection(url, user, password);
- // 打开事物
- con.setAutoCommit(false);
- // SQL执行语句
- String exeSql = "";
- // 插入值
- String insertValue = "";
- // 插入的列名
- String insertColumName = "";
- // 更新的SQL语句
- String updateSQL = "";
- // SQL操作语句(用来判断数据库里就没有记录等)
- String optionSql = "";
- // 当前记录的ID值
- String rowID = "";
- // ID的列名
- String idName = "";
- int flag = 0;
-
- // ------------读取XML文件,获得document对象.
- SAXReader reader = new SAXReader();
- reader.setEncoding("Utf-8");
- Document document = null;
- document = reader.read(new File(filePath));
- // -----------获取文档的根节点.
- Element root = document.getRootElement();
-
- // 遍历所有table节点
- for (Iterator rootIter = root.elementIterator("Table"); rootIter
- .hasNext();) {
- Element tableNode = (Element) rootIter.next();
- // 得到table节点的Name属性值,用来判断属于哪一个数据库表
- String tableName = tableNode.attributeValue("Name");
- // 打印数据库表名
- System.out.println("数据库表名" + tableName);
- // 取得table节点下的所有子节点,并且遍历
-
- for (Iterator tableIter = tableNode.elementIterator(); tableIter
- .hasNext();) {
- Element row = (Element) tableIter.next();
- // 得到每一条记录里面的数据
- for (Iterator rowIter = row.elementIterator(); rowIter
- .hasNext();) {
-
- flag++;
-
- Element col = (Element) rowIter.next();
-
- if (flag == 1) {
-
- rowID = col.getText();
- idName = col.getName();
-
- }
-
- // 得到当前节点的名字,用来判断属于数据库中的哪一列
- String columName = col.getName();
- // 得到该节点的字段类型
- String columType = col.attributeValue("type");
-
- // 得到当前节点的值,用来插入数据库中
- // 列值。默认为空字符
- String columValue = "";
- // 判断节点字段的类型,然后转换字段类型
- if (columType.equals("date")) {
- columValue = "to_date(substr('" + col.getText()
- + "',1,10),'yyyy-MM-dd')";
-
- } else {
- columValue = "'" + col.getText() + "'";
- // System.out.println(columType);
- }
-
- // 判断是否为最后一个,如果是则不加','号
- if (rowIter.hasNext()) {
- insertValue += columValue + ",";
- insertColumName += columName + ",";
-
- } else {
- insertValue += columValue;
- insertColumName += columName;
-
- }
- // 判断是否为最后一个,如果是则不加','号
- if (rowIter.hasNext()) {
- updateSQL += columName + "=" + columValue + ",";
- } else {
-
- updateSQL += columName + "=" + columValue;
- }
-
- }
-
- optionSql = "select * from " + tableName + " where "
- + idName + "='" + rowID + "'";
- PreparedStatement optionps = con
- .prepareStatement(optionSql);
- ResultSet opRS = optionps.executeQuery();
-
- System.out.println(optionSql);
-
- // 假如数据库里没有记录则插入
- if (!opRS.next()) {
-
- exeSql = "insert into " + tableName + "("
- + insertColumName + ")" + "values("
- + insertValue + ")";
- PreparedStatement ps = con.prepareStatement(exeSql);
- System.out.println("插入");
- System.out.println(exeSql);
- ps.execute();
- opRS.close();
- optionps.close();
- ps.close();
-
- } else {
- // 假如数据库里有记录则更新
- exeSql = "update " + tableName + " set " + updateSQL
- + " where " + idName + "=" + "'" + rowID + "'";
- PreparedStatement ps = con.prepareStatement(exeSql);
- System.out.println("更新");
- System.out.println(exeSql);
- ps.execute();
- opRS.close();
- optionps.close();
- ps.close();
-
- }
- // 插入完一条记录以后清空记录信息
- exeSql = "";
- insertValue = "";
- insertColumName = "";
- updateSQL = "";
- optionSql = "";
- rowID = "";
- idName = "";
- flag = 0;
-
- }
- }
- con.commit();
- con.close();
- System.out.println("============导入完成============");
-
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- importXml("C:/ProgramData/MySQL/MySQL Server 5.5/data/webmagic/Badges.xml");
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/main.iml b/crawler/dailyScheduledCrawler/fetch_networks/src/main/main.iml
deleted file mode 100644
index 489e48878..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/main.iml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/log4j.xml b/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/log4j.xml
deleted file mode 100644
index 157cc96c0..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,77 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/spring/applicationContext-myBatis.xml b/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/spring/applicationContext-myBatis.xml
deleted file mode 100644
index 5956c8bc5..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/spring/applicationContext-myBatis.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/spring/applicationContext.xml b/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/spring/applicationContext.xml
deleted file mode 100644
index 7d83874af..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/spring/applicationContext.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/testDBlogger.xml b/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/testDBlogger.xml
deleted file mode 100644
index bc2391d0b..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/main/resources/testDBlogger.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/CSSPathExtractorTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/CSSPathExtractorTest.java
deleted file mode 100644
index d3d5fb8a2..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/CSSPathExtractorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package net.trustie.webmagic.one.extractor;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Set;
-
-import net.trustie.webmagic.extrator.CSSPathExtractor;
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.XPathExtractor;
-
-import org.jsoup.Jsoup;
-
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-
-public class CSSPathExtractorTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- String urlStr = "http://www.freecode.com/";
- String doc = "";
- try {
- URL url = new URL(urlStr);
- doc = Jsoup.parse(url, 100000).html();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- Html html = new Html(UrlUtils.fixAllRelativeHrefs(doc,
- "http://www.freecode.com/"));
- Extractor extractor = new CSSPathExtractor("div.release h2.release-head>a");
- Set urlSet = extractor.extract(html);
- System.out.println(urlSet.size());
- for (String url : urlSet) {
- System.out.println(url);
- }
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/XPathExtractorTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/XPathExtractorTest.java
deleted file mode 100644
index 5c3701866..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/XPathExtractorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package net.trustie.webmagic.one.extractor;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Set;
-
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.XPathExtractor;
-
-import org.jsoup.Jsoup;
-
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-import us.codecraft.xsoup.Xsoup;
-
-public class XPathExtractorTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- String urlStr = "http://www.freecode.com/";
- String doc="";
- try {
- URL url = new URL(urlStr);
- doc= Jsoup.parse(url, 100000).html();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- Html html = new Html(UrlUtils.fixAllRelativeHrefs(doc,
- "http://www.freecode.com/"));
- Extractor extractor = new XPathExtractor("//div[@class='release']/h2[@class='release-head']/a/@href");
- Set urlSet = extractor.extract(html);
- System.out.println(urlSet.size());
- for (String url : urlSet) {
- System.out.println(url);
- }
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/FileReaderTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/FileReaderTest.java
deleted file mode 100644
index a57dce73a..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/FileReaderTest.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.trustie.webmagic.others;
-
-import net.trustie.webmagic.utils.FileReader;
-
-public class FileReaderTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- System.out.println(FileReader.readFile("./sql/list_html.sql"));
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StaticTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StaticTest.java
deleted file mode 100644
index f3e55f06c..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StaticTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package net.trustie.webmagic.others;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import net.trustie.webmagic.one.Configure;
-import net.trustie.webmagic.one.DetailHtmlProcessor;
-import net.trustie.webmagic.one.DetailHtmlSubProcessor;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-
-public class StaticTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- Configure conf = new Configure("ossean");
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
- Site site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- List urlList = new ArrayList();
- urlList.add("http://localhost/open_source_projects?page=1");
- while(true){
- Spider
- .create(new DetailHtmlProcessor(site))
- .addPipeline(new ConsolePipeline()).setScheduler(new SimpleScheduler()).thread(conf.getThreadNum()).startUrls(urlList).run();
- }
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StringTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StringTest.java
deleted file mode 100644
index 035bd4279..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StringTest.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.trustie.webmagic.others;
-
-public class StringTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- String tmp = "abc";
- System.out.println(tmp.toString());
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/spider/SpiderTet.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/spider/SpiderTet.java
deleted file mode 100644
index ecbfba9cb..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/spider/SpiderTet.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package net.trustie.webmagic.spider;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-public class SpiderTet {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- List urls = new ArrayList();
- urls.add("http://www.oschina.net/question?catalog=1&show=active&p=1");
- urls.add("http://www.oschina.net/question?catalog=1&show=active&p=2");
- urls.add("http://www.oschina.net/question?catalog=1&show=active&p=3");
- Spider spider = Spider.create(new PageProcessor() {
-
- @Override
- public void process(Page arg0) {
- // TODO Auto-generated method stub
- System.out.println(1);
- }
-
- @Override
- public Site getSite() {
- // TODO Auto-generated method stub
- return null;
- }
- }).addPipeline(new ConsolePipeline()).thread(3).startUrls(urls);
- spider.run();
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/BugInfoDaoTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/BugInfoDaoTest.java
deleted file mode 100644
index 631764739..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/BugInfoDaoTest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package net.trustie.webmagic.test.dao;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-
-import javax.annotation.Resource;
-
-/**
- * User: cairne Date: 13-5-13 Time: 下午8:33
- */
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(locations = { "classpath:/spring/applicationContext-*.xml" })
-public class BugInfoDaoTest {
-
- @Ignore
- @Test
- public void test() {
- /*
- * OpenStackBugInfo bugInfo = new OpenStackBugInfo();
- * bugInfo.setSource("a"); try { final int add =
- * bugInfoDAO.add(bugInfo); System.out.println(add); } catch (Exception
- * e) { e.printStackTrace(); }
- */
- }
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/MybatisTest.java b/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/MybatisTest.java
deleted file mode 100644
index da00e1023..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/MybatisTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package net.trustie.webmagic.test.dao;
-
-import java.util.List;
-
-import javax.annotation.Resource;
-
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.model.ListHtml;
-
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-@Component
-public class MybatisTest {
- @Resource
- private ListHtmlDao listHtmlDao;
- public List getBatch(){
- return listHtmlDao.getBatch("freecode_html_list", 0 , 4);
- }
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "spring/applicationContext*.xml");
- MybatisTest mbt = applicationContext.getBean(MybatisTest.class);
- List list= mbt.getBatch();
- for(ListHtml html : list){
- System.out.println(html.toString());
- }
- System.out.println(list.size());
- }
-
-}
diff --git a/crawler/dailyScheduledCrawler/fetch_networks/src/test/test.iml b/crawler/dailyScheduledCrawler/fetch_networks/src/test/test.iml
deleted file mode 100644
index c2700a337..000000000
--- a/crawler/dailyScheduledCrawler/fetch_networks/src/test/test.iml
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/crawler/daily_scheduler/.classpath b/crawler/daily_scheduler/.classpath
deleted file mode 100644
index c3d80006f..000000000
--- a/crawler/daily_scheduler/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-
-
-
-
-
diff --git a/crawler/daily_scheduler/.settings/org.eclipse.jdt.core.prefs b/crawler/daily_scheduler/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 7341ab168..000000000
--- a/crawler/daily_scheduler/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,11 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.source=1.7
diff --git a/crawler/daily_scheduler/bin/.gitignore b/crawler/daily_scheduler/bin/.gitignore
deleted file mode 100644
index c2d9872a1..000000000
--- a/crawler/daily_scheduler/bin/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/com/
diff --git a/crawler/daily_scheduler/delete.sh b/crawler/daily_scheduler/delete.sh
deleted file mode 100644
index d45d2ad99..000000000
--- a/crawler/daily_scheduler/delete.sh
+++ /dev/null
@@ -1 +0,0 @@
-ps -aux|grep oschina_question |grep -v grep|cut -c 9-15|xargs kill -9
\ No newline at end of file
diff --git a/crawler/daily_scheduler/lib/sigar.jar b/crawler/daily_scheduler/lib/sigar.jar
deleted file mode 100644
index 58c733c6a..000000000
Binary files a/crawler/daily_scheduler/lib/sigar.jar and /dev/null differ
diff --git a/crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java b/crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java
deleted file mode 100644
index 0a328785c..000000000
--- a/crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java
+++ /dev/null
@@ -1,208 +0,0 @@
-package com.ossean.crawler.dailyScheduler;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.hyperic.sigar.Mem;
-import org.hyperic.sigar.Sigar;
-import org.hyperic.sigar.SigarException;
-
-class MyTask implements Runnable{
- private SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式
- String cmdPrefix = "sh bin/";
- String cmd;
- long rotateInterval = 1000 * 60 * 60 * 1;// schedule some tasks every 1 hours for memory limit
- int memoPerTask = 800;//the memory a task needs
- int memoLeft = 600;
- Runtime rt;
- Process process;
- BufferedReader bufferedReader;
- Queue tasks = new LinkedList();
- //String cmd = cmdPrefix + "bytes.sh";
-
- public void schedule(){
- long lastSchedule = System.currentTimeMillis();//上一次的调度时间
- long now;//此次调度结束的时间
- long sleepTime;//距离下次调度要睡眠的时间
-
- while(true){
- System.out.println("----------------------------------------------------- ");
- System.out.println(">>>>>schedule start @ " + df.format(new Date()));
- // step 1: read tasks to be scheduled today
- readTasks();
-
- // step 2: do job
- doSchedule();
- System.out.println(">>>>>schedule done @ " + df.format(new Date()));
- now = System.currentTimeMillis();
- long period = (now - lastSchedule);
- if(period < 24 * 60 * 60 * 1000){
- sleepTime = (24 * 60 * 60 * 1000) - period;
- try {
- System.out.println(">>>>>sleep " + sleepTime /1000 + "s @ " + df.format(new Date()));
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- System.out.println("----------------------------------------------------- ");
- lastSchedule = System.currentTimeMillis();
-
- }
- }
- private void doSchedule() {
-
- while(true){
- //System.out.println("check out the free memory...");
- // step 3.1: whether enough memory can be used
- long freeMemo = 0;
- try {
- Sigar sigar = new Sigar();
- Mem mem = sigar.getMem();
- freeMemo = mem.getFree() / (1024 * 1024);
-
- //System.out.println("free memo: " + freeMemo + " @ " + df.format(new Date()));
- } catch (SigarException e1) {
- // TODO Auto-generated catch block
- e1.printStackTrace();
- }
- if(freeMemo < (memoPerTask + memoLeft )){
- //System.out.println("not enough memo");
- try {
- Thread.sleep(60 * 20 * 1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- continue;
- }
- //step 3.2: schedule some task
- int scheduleNum = (int)((freeMemo - memoLeft ) / memoPerTask);
- while(scheduleNum-- > 0 && tasks.size() > 0){
- String task = tasks.poll().trim();
- System.out.print( task + " @ " + df.format(new Date()) + " ,");
- //kill the task scheduled yesterday if exists
- killProcess(task);
- this.cmd = this.cmdPrefix + task+".sh";
- rt = Runtime.getRuntime();
- try {
- process = rt.exec(cmd,new String[]{},new File("/home/pdl/programs/dailyScheduledCrawler/fetch_networks/"));
- bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024);
- process.waitFor();
- String line;
- // System.out.println("the output of script");
- // while((line = bufferedReader.readLine()) != null){
- // System.out.println(line);
- // }
- // System.out.println(">>>>>>>> schedule done " + df.format(new Date()));
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- }
-
- if(tasks.size() == 0){
- //System.out.println("today's schedule is over "+ df.format(new Date()));
- break;
- }
-
- try {
- //System.out.println("sleep for rotatenterval time");
- System.out.println( );
- Thread.sleep(this.rotateInterval);
- //System.out.println("sleep over");
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
- private void killProcess(String task) {
-
- try{
- //System.out.println(" kill process:" +task);
- rt = Runtime.getRuntime();
- String deleteStr = "ps -aux|grep " + task + " |grep -v grep|cut -c 9-15|xargs kill -9" ;
- FileWriter fileWritter = new FileWriter(new File("delete.sh").getName(),false);
- BufferedWriter bufferWritter = new BufferedWriter(fileWritter);
- bufferWritter.write(deleteStr);
- bufferWritter.close();
-
- process = rt.exec("sh delete.sh");
- bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024);
- process.waitFor();
- String line;
- // System.out.println("the output of kill:");
- // while((line = bufferedReader.readLine()) != null){
- // System.out.println(line);
- //}
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- }
- private void readTasks() {
- String fileName = "tasks.txt";
- File file = new File(fileName);
- BufferedReader reader = null;
- try {
- //System.out.println("begain to read task list from file tasks.txt" + " @ " + df.format(new Date()));
- reader = new BufferedReader(new FileReader(file));
- String tempString = null;
- // System.out.println("tasks are : >>>>>");
- while ((tempString = reader.readLine()) != null) {
- // System.out.println(tempString);
- tasks.add(tempString);
- }
- reader.close();
- // System.out.println("done to read task " + " @ " + df.format(new Date()));
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (reader != null) {
- try {
- reader.close();
- } catch (IOException e1) {
- }
- }
- }
- }
- @Override
- public void run() {
- schedule();
- }
-}
-public class DailyScheduler {
- static ScheduledThreadPoolExecutor stpe = null;
- public static void main(String[] args) {
-
- //stpe = new ScheduledThreadPoolExecutor(2);
- MyTask task = new MyTask();
- // System.out.println("here we go");
- task.schedule();
- //stpe.scheduleAtFixedRate(task, 0, 24*60*60, TimeUnit.SECONDS);
-
- }
-}
diff --git a/crawler/daily_scheduler/tasks.txt b/crawler/daily_scheduler/tasks.txt
deleted file mode 100644
index 365f67d11..000000000
--- a/crawler/daily_scheduler/tasks.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-csdn_ask
-iteye_blog
-cnblogs_q_unsolved
-neitui
-code_project
-cnblogs_news
-oschina_question
-51cto_blog
-csdn_topic
-dewen_question
-cnblogs_q_solved
diff --git a/crawler/daily_scheduler/tasks.txt.backup b/crawler/daily_scheduler/tasks.txt.backup
deleted file mode 100644
index 5f60205f3..000000000
--- a/crawler/daily_scheduler/tasks.txt.backup
+++ /dev/null
@@ -1,19 +0,0 @@
-csdn_ask
-slashdot
-iteye_blog
-cnblogs_q_unsolved
-neitui
-ibm_post
-code_project
-cnblogs_news
-oschina_question
-51cto_blog
-csdn_topic
-softpedia
-bytes
-oschina_project
-sourceforge
-openhub
-csdn_blog
-dewen_question
-cnblogs_q_solved
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/.idea/compiler.xml b/crawler/moreSmarterCrawler/.idea/compiler.xml
deleted file mode 100644
index 61a9130cd..000000000
--- a/crawler/moreSmarterCrawler/.idea/compiler.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/.idea/misc.xml b/crawler/moreSmarterCrawler/.idea/misc.xml
deleted file mode 100644
index 348936d64..000000000
--- a/crawler/moreSmarterCrawler/.idea/misc.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/.idea/modules.xml b/crawler/moreSmarterCrawler/.idea/modules.xml
deleted file mode 100644
index f195522af..000000000
--- a/crawler/moreSmarterCrawler/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/.idea/moreSmarterCrawler.iml b/crawler/moreSmarterCrawler/.idea/moreSmarterCrawler.iml
deleted file mode 100644
index d6ebd4805..000000000
--- a/crawler/moreSmarterCrawler/.idea/moreSmarterCrawler.iml
+++ /dev/null
@@ -1,9 +0,0 @@
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/.idea/workspace.xml b/crawler/moreSmarterCrawler/.idea/workspace.xml
deleted file mode 100644
index a571633df..000000000
--- a/crawler/moreSmarterCrawler/.idea/workspace.xml
+++ /dev/null
@@ -1,233 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
- DEFINITION_ORDER
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1503302159478
-
-
- 1503302159478
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.classpath b/crawler/moreSmarterCrawler/fetch_networks/.classpath
deleted file mode 100644
index 07152b10b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.classpath
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/.name b/crawler/moreSmarterCrawler/fetch_networks/.idea/.name
deleted file mode 100644
index d2c26254d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/.name
+++ /dev/null
@@ -1 +0,0 @@
-fetchnetworks
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/compiler.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/compiler.xml
deleted file mode 100644
index 2f600634a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/compiler.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources.local.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources.local.xml
deleted file mode 100644
index 531333f7f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources.local.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-
-
-
-
-
- #@
- `
-
-
- master_key
- root
- *:crawler
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources.xml
deleted file mode 100644
index b8c80870b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
- mysql
- true
- com.mysql.jdbc.Driver
- jdbc:mysql://localhost:3306/crawler
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources/e2f306f5-80be-46ee-9e83-ac1795b7fdb7.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources/e2f306f5-80be-46ee-9e83-ac1795b7fdb7.xml
deleted file mode 100644
index 2bf12bedd..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/dataSources/e2f306f5-80be-46ee-9e83-ac1795b7fdb7.xml
+++ /dev/null
@@ -1,278 +0,0 @@
-
-
-
-
-
- 1
- 1
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- mediumtext|0
-
-
- 4
- datetime|0
-
-
- 5
- varchar(255)|0
-
-
- 6
- varchar(255)|0
-
-
- 7
- varchar(255)|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- mediumtext|0
-
-
- 4
- datetime|0
-
-
- 5
- varchar(255)|0
-
-
- 6
- varchar(255)|0
-
-
- 7
- varchar(255)|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- bigint(20)|0
-
-
- 4
- datetime|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- mediumtext|0
-
-
- 4
- datetime|0
-
-
- 5
- varchar(255)|0
-
-
- 6
- varchar(255)|0
-
-
- 7
- varchar(255)|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- int(11)|0
-
-
- 4
- datetime|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- mediumtext|0
-
-
- 4
- datetime|0
-
-
- 5
- varchar(255)|0
-
-
- 6
- varchar(255)|0
-
-
- 7
- varchar(255)|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- mediumtext|0
-
-
- 4
- datetime|0
-
-
- 5
- varchar(255)|0
-
-
- 6
- varchar(255)|0
-
-
- 7
- varchar(255)|0
-
-
- 1
- id
- 1
-
-
- 1
- int(11)|0
- 1
- 1
-
-
- 2
- varchar(255)|0
-
-
- 3
- bigint(20)|0
-
-
- 4
- datetime|0
-
-
- 1
- id
- 1
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/encodings.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/encodings.xml
deleted file mode 100644
index b26911bd0..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/encodings.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml
deleted file mode 100644
index 30ff5cb79..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__cglib_cglib_nodep_2_1_3.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__cglib_cglib_nodep_2_1_3.xml
deleted file mode 100644
index 55692b94d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__cglib_cglib_nodep_2_1_3.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_alibaba_fastjson_1_2_3.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_alibaba_fastjson_1_2_3.xml
deleted file mode 100644
index 79b4a8031..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_alibaba_fastjson_1_2_3.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml
deleted file mode 100644
index 0119075af..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_annotations_2_4_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml
deleted file mode 100644
index 072f8ef49..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_core_2_4_4.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml
deleted file mode 100644
index efdcf45c6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_fasterxml_jackson_core_jackson_databind_2_4_4.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_github_dreamhead_moco_core_0_10_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_github_dreamhead_moco_core_0_10_0.xml
deleted file mode 100644
index f01430169..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_github_dreamhead_moco_core_0_10_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_google_guava_guava_14_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_google_guava_guava_14_0.xml
deleted file mode 100644
index 86eb63f34..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_google_guava_guava_14_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml
deleted file mode 100644
index 5347a68fa..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_codec_commons_codec_1_6.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_codec_commons_codec_1_6.xml
deleted file mode 100644
index e8a6a9f91..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_codec_commons_codec_1_6.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml
deleted file mode 100644
index 3caee7e54..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_3.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_3.xml
deleted file mode 100644
index ebd31e46a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_3.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml
deleted file mode 100644
index 7b5b3b758..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml
deleted file mode 100644
index 2ec837671..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml
deleted file mode 100644
index 01c7b8ee6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml
deleted file mode 100644
index ab2072126..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml
deleted file mode 100644
index 14681ee84..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__dom4j_dom4j_1_6_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_3_5_2_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_3_5_2_Final.xml
deleted file mode 100644
index 7de2ea925..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_3_5_2_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_buffer_4_0_24_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_buffer_4_0_24_Final.xml
deleted file mode 100644
index d3ce414ea..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_buffer_4_0_24_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_codec_4_0_24_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_codec_4_0_24_Final.xml
deleted file mode 100644
index 271fe7684..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_codec_4_0_24_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_codec_http_4_0_24_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_codec_http_4_0_24_Final.xml
deleted file mode 100644
index 6f4e0b73c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_codec_http_4_0_24_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_common_4_0_24_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_common_4_0_24_Final.xml
deleted file mode 100644
index 0ff3df037..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_common_4_0_24_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_handler_4_0_24_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_handler_4_0_24_Final.xml
deleted file mode 100644
index b8a9a4792..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_handler_4_0_24_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_transport_4_0_24_Final.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_transport_4_0_24_Final.xml
deleted file mode 100644
index 3e52df61c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__io_netty_netty_transport_4_0_24_Final.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml
deleted file mode 100644
index 679e09a1d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__javax_servlet_servlet_api_2_5.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__junit_junit_4_7.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__junit_junit_4_7.xml
deleted file mode 100644
index 4402995f8..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__junit_junit_4_7.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__log4j_log4j_1_2_17.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__log4j_log4j_1_2_17.xml
deleted file mode 100644
index e383c1bf1..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__log4j_log4j_1_2_17.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml
deleted file mode 100644
index 17b6e606c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_java_dev_jna_jna_3_4_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_java_dev_jna_jna_3_4_0.xml
deleted file mode 100644
index 4aa2e5789..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_java_dev_jna_jna_3_4_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_java_dev_jna_platform_3_4_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_java_dev_jna_platform_3_4_0.xml
deleted file mode 100644
index ed237590b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_java_dev_jna_platform_3_4_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml
deleted file mode 100644
index 666695874..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_cssparser_cssparser_0_9_13.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_cssparser_cssparser_0_9_13.xml
deleted file mode 100644
index 9df8cc035..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_cssparser_cssparser_0_9_13.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_2_14.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_2_14.xml
deleted file mode 100644
index 0b9cff8ef..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_2_14.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_core_js_2_14.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_core_js_2_14.xml
deleted file mode 100644
index f97c0b74f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_htmlunit_htmlunit_core_js_2_14.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_nekohtml_nekohtml_1_9_20.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_nekohtml_nekohtml_1_9_20.xml
deleted file mode 100644
index cc26369d4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__net_sourceforge_nekohtml_nekohtml_1_9_20.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_commons_commons_exec_1_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_commons_commons_exec_1_1.xml
deleted file mode 100644
index 3c118df01..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_commons_commons_exec_1_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml
deleted file mode 100644
index 32bfe3b20..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_4.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_4.xml
deleted file mode 100644
index 7a0e9f6de..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_4.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml
deleted file mode 100644
index 1c1496328..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_3_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_3_2.xml
deleted file mode 100644
index 6894e0460..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_apache_httpcomponents_httpmime_4_3_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_assertj_assertj_core_1_7_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_assertj_assertj_core_1_7_0.xml
deleted file mode 100644
index 3c2666683..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_assertj_assertj_core_1_7_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_8_1_14_v20131031.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_8_1_14_v20131031.xml
deleted file mode 100644
index 3909906ec..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_http_8_1_14_v20131031.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_8_1_14_v20131031.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_8_1_14_v20131031.xml
deleted file mode 100644
index 1760bf6cc..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_io_8_1_14_v20131031.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_8_1_14_v20131031.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_8_1_14_v20131031.xml
deleted file mode 100644
index f0b08c5fa..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_util_8_1_14_v20131031.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_websocket_8_1_14_v20131031.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_websocket_8_1_14_v20131031.xml
deleted file mode 100644
index f470623d4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_eclipse_jetty_jetty_websocket_8_1_14_v20131031.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_freemarker_freemarker_2_3_21.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_freemarker_freemarker_2_3_21.xml
deleted file mode 100644
index 3f988f75b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_freemarker_freemarker_2_3_21.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_json_json_20141113.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_json_json_20141113.xml
deleted file mode 100644
index 2f486e750..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_json_json_20141113.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml
deleted file mode 100644
index 2972f2e5e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml
deleted file mode 100644
index 75070eb48..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml
deleted file mode 100644
index 0ff136f4a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_api_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_api_2_42_2.xml
deleted file mode 100644
index a8b18af22..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_api_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_chrome_driver_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_chrome_driver_2_42_2.xml
deleted file mode 100644
index dacedffb8..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_chrome_driver_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_firefox_driver_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_firefox_driver_2_42_2.xml
deleted file mode 100644
index 11ec5a6d7..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_firefox_driver_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_htmlunit_driver_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_htmlunit_driver_2_42_2.xml
deleted file mode 100644
index 056170272..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_htmlunit_driver_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_ie_driver_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_ie_driver_2_42_2.xml
deleted file mode 100644
index f122fefd5..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_ie_driver_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_java_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_java_2_42_2.xml
deleted file mode 100644
index b73545f8a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_java_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_remote_driver_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_remote_driver_2_42_2.xml
deleted file mode 100644
index 26e06f0a7..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_remote_driver_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_safari_driver_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_safari_driver_2_42_2.xml
deleted file mode 100644
index fc285a790..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_safari_driver_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_support_2_42_2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_support_2_42_2.xml
deleted file mode 100644
index 90a155a5f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_seleniumhq_selenium_selenium_support_2_42_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml
deleted file mode 100644
index 1e672600f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_7.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_7.xml
deleted file mode 100644
index 65d0ae68d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_7.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml
deleted file mode 100644
index d84b574ff..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml
deleted file mode 100644
index 1d6ab8693..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml
deleted file mode 100644
index 90a83383f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml
deleted file mode 100644
index a240c8b39..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml
deleted file mode 100644
index 59bc41161..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml
deleted file mode 100644
index 4b9e1dad4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml
deleted file mode 100644
index e037dcfa7..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_test_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_test_3_1_1_RELEASE.xml
deleted file mode 100644
index 676777e71..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_test_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml
deleted file mode 100644
index f7edb5d88..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_w3c_css_sac_1_3.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_w3c_css_sac_1_3.xml
deleted file mode 100644
index af3d41461..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_w3c_css_sac_1_3.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_webbitserver_webbit_0_4_14.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_webbitserver_webbit_0_4_14.xml
deleted file mode 100644
index 1af445796..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__org_webbitserver_webbit_0_4_14.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml
deleted file mode 100644
index 8a6022ea5..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xalan_serializer_2_7_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xalan_serializer_2_7_1.xml
deleted file mode 100644
index 8891e3c18..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xalan_serializer_2_7_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xalan_xalan_2_7_1.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xalan_xalan_2_7_1.xml
deleted file mode 100644
index 4149f4f6c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xalan_xalan_2_7_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xerces_xercesImpl_2_4_0.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xerces_xercesImpl_2_4_0.xml
deleted file mode 100644
index 6f3f5c50d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xerces_xercesImpl_2_4_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml
deleted file mode 100644
index c36e71714..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/libraries/Maven__xml_apis_xml_apis_1_0_b2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/misc.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/misc.xml
deleted file mode 100644
index ff20635a5..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/misc.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/modules.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/modules.xml
deleted file mode 100644
index d269f2278..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/vcs.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/vcs.xml
deleted file mode 100644
index c2365ab11..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/vcs.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.idea/workspace.xml b/crawler/moreSmarterCrawler/fetch_networks/.idea/workspace.xml
deleted file mode 100644
index 722d8e48b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.idea/workspace.xml
+++ /dev/null
@@ -1,1644 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
- DEFINITION_ORDER
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Android > Lint > Correctness
-
-
- Gradle
-
-
- Kotlin
-
-
- Maven
-
-
- OSGi
-
-
- Probable bugsGradle
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- project
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- $USER_HOME$/.subversion
-
-
-
-
- 1484724096170
-
-
- 1484724096170
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java
- 131
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Spring (fetchnetworks)|Spring
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
- fetchnetworks
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
- Maven: aopalliance:aopalliance:1.0
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.project b/crawler/moreSmarterCrawler/fetch_networks/.project
deleted file mode 100644
index 43f566a3c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.project
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
- fetchnetworks
-
-
-
-
-
- org.eclipse.jdt.core.javabuilder
-
-
-
-
- org.maven.ide.eclipse.maven2Builder
-
-
-
-
- org.eclipse.m2e.core.maven2Builder
-
-
-
-
-
- org.maven.ide.eclipse.maven2Nature
- org.eclipse.jdt.core.javanature
- org.eclipse.m2e.core.maven2Nature
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs b/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index 839d647ee..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
-encoding/=UTF-8
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs b/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs
deleted file mode 100644
index 5a0ad22d2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs
+++ /dev/null
@@ -1,2 +0,0 @@
-eclipse.preferences.version=1
-line.separator=\n
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs b/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 308d3cd9a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,14 +0,0 @@
-#Sun Jan 04 15:44:05 CST 2015
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.7
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.7
diff --git a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs b/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs
deleted file mode 100644
index f897a7f1c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-activeProfiles=
-eclipse.preferences.version=1
-resolveWorkspaceProjects=true
-version=1
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/51cto_blog.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/51cto_blog.sh
deleted file mode 100644
index 13521a7a7..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/51cto_blog.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='51cto_blog'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/apache.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/apache.sh
deleted file mode 100644
index 2a84a2205..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/apache.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='apache'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/bytes.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/bytes.sh
deleted file mode 100644
index 00ee7e74c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/bytes.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='bytes'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_news.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_news.sh
deleted file mode 100644
index f665d764b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_news.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='cnblogs_news'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_q_solved.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_q_solved.sh
deleted file mode 100644
index a2f031e9d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_q_solved.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='cnblogs_q_solved'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh
deleted file mode 100644
index 4420e3298..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='cnblogs_q_unsolved'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/code_project.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/code_project.sh
deleted file mode 100644
index 75ad87897..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/code_project.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='code_project'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_ask.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_ask.sh
deleted file mode 100644
index 29cb5da85..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_ask.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='csdn_ask'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_blog.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_blog.sh
deleted file mode 100644
index 1f35c302f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_blog.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='csdn_blog'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_topic.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_topic.sh
deleted file mode 100644
index c95465e51..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/csdn_topic.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='csdn_topic'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx256m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/dewen_question.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/dewen_question.sh
deleted file mode 100644
index 3ff93af93..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/dewen_question.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='dewen_question'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/freecode.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/freecode.sh
deleted file mode 100644
index fc80a05e1..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/freecode.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='freecode'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/gna.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/gna.sh
deleted file mode 100644
index e4ac9d731..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/gna.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='gna'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/ibm_post.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/ibm_post.sh
deleted file mode 100644
index 79249e1b5..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/ibm_post.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='ibm_post'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/ibm_project.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/ibm_project.sh
deleted file mode 100644
index ac537452b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/ibm_project.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='ibm_project'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/iteye_ask.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/iteye_ask.sh
deleted file mode 100644
index 8844b8f0c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/iteye_ask.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='iteye_ask'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/iteye_blog.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/iteye_blog.sh
deleted file mode 100644
index 38cdddfde..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/iteye_blog.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='iteye_blog'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/lagou.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/lagou.sh
deleted file mode 100644
index 36df3cf54..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/lagou.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='lagou'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/linuxtone.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/linuxtone.sh
deleted file mode 100644
index 3ae829b9f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/linuxtone.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='linuxtone'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/lupaworld.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/lupaworld.sh
deleted file mode 100644
index 771c261ac..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/lupaworld.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='lupaworld'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/neitui.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/neitui.sh
deleted file mode 100644
index eae7e58cb..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/neitui.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='neitui'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/openhub.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/openhub.sh
deleted file mode 100644
index fddcedcd1..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/openhub.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='openhub'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/oschina_project.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/oschina_project.sh
deleted file mode 100644
index c5e71be13..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/oschina_project.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='oschina_project'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/oschina_question.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/oschina_question.sh
deleted file mode 100644
index 7eb57ee13..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/oschina_question.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='oschina_question'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/phpchina.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/phpchina.sh
deleted file mode 100644
index ed6173db4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/phpchina.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='phpchina'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/log4j.xml b/crawler/moreSmarterCrawler/fetch_networks/bin/resources/log4j.xml
deleted file mode 100644
index fb9cbc5a6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/log4j.xml
+++ /dev/null
@@ -1,79 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml b/crawler/moreSmarterCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml
deleted file mode 100644
index ada70d40f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 21600000
-
-
-
- 21600000
-
-
-
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/spring/applicationContext.xml b/crawler/moreSmarterCrawler/fetch_networks/bin/resources/spring/applicationContext.xml
deleted file mode 100644
index 7d83874af..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/spring/applicationContext.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/testDBlogger.xml b/crawler/moreSmarterCrawler/fetch_networks/bin/resources/testDBlogger.xml
deleted file mode 100644
index bc2391d0b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/resources/testDBlogger.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/sample_start.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/sample_start.sh
deleted file mode 100644
index 492857d57..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/sample_start.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='freecode'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/slashdot.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/slashdot.sh
deleted file mode 100644
index 786690777..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/slashdot.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='slashdot'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/softpedia.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/softpedia.sh
deleted file mode 100644
index db9e7db3b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/softpedia.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='softpedia'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/sourceforge.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/sourceforge.sh
deleted file mode 100644
index 67e33209c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/sourceforge.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-SITE='sourceforge'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/stackoverflow.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/stackoverflow.sh
deleted file mode 100644
index 22913ef36..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/stackoverflow.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-#!!!!!!!!!!!!与sites中对应文件文件名相同,不包括扩展名
-SITE='stackoverflow'
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-find ./target/classes/spring |xargs rm -f -r
-find ./target/classes/sites |xargs rm -f -r
-
-tmp='./bin/resources'
-tmp='./sites':$tmp
-tmp='./target/classes':$tmp
-tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn48m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
-#-XX:MinPermSize=128m"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
-java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
-#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
diff --git a/crawler/moreSmarterCrawler/fetch_networks/bin/start_all.sh b/crawler/moreSmarterCrawler/fetch_networks/bin/start_all.sh
deleted file mode 100644
index e901cc634..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/bin/start_all.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-sh bin/cnblogs_news.sh
-sh bin/cnblogs_q_solved.sh
-sh bin/cnblogs_q_unsolved.sh
-sh bin/csdn_ask.sh
-sh bin/csdn_blog.sh
-sh bin/csdn_topic.sh
-sh bin/dewen_question.sh
-sh bin/iteye_ask.sh
-sh bin/openhub.sh
-sh bin/oschina_project.sh
-sh bin/oschina_question.sh
-sh bin/sourceforge.sh
-sh bin/stackoverflow.sh
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/fetchnetworks.iml b/crawler/moreSmarterCrawler/fetch_networks/fetchnetworks.iml
deleted file mode 100644
index 536780d53..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/fetchnetworks.iml
+++ /dev/null
@@ -1,111 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/lib/webmagic-core-fix.jar b/crawler/moreSmarterCrawler/fetch_networks/lib/webmagic-core-fix.jar
deleted file mode 100644
index 39d45c42e..000000000
Binary files a/crawler/moreSmarterCrawler/fetch_networks/lib/webmagic-core-fix.jar and /dev/null differ
diff --git a/crawler/moreSmarterCrawler/fetch_networks/lib/webmagic-extension-fix.jar b/crawler/moreSmarterCrawler/fetch_networks/lib/webmagic-extension-fix.jar
deleted file mode 100644
index 7a204b496..000000000
Binary files a/crawler/moreSmarterCrawler/fetch_networks/lib/webmagic-extension-fix.jar and /dev/null differ
diff --git a/crawler/moreSmarterCrawler/fetch_networks/ossean-bak.xml b/crawler/moreSmarterCrawler/fetch_networks/ossean-bak.xml
deleted file mode 100644
index cac15888e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/ossean-bak.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-
-Spider
-1
-one
-5
-1
-true
-20000
-5
-30000
-5000
-5
-Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-10000
-http://www.oschina.net/question/
-http://www\.oschina\.net/question\?catalog=[12]\&show=active\&p=\d+
-http://www\.oschina\.net/question/\d+_\d+
-0
-10
-10000
-html_test
-target_url
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/pom.xml b/crawler/moreSmarterCrawler/fetch_networks/pom.xml
deleted file mode 100644
index cc181748b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/pom.xml
+++ /dev/null
@@ -1,202 +0,0 @@
-
-
-
- 4.0.0
-
- net.trustie
- fetchnetworks
- 1.0-SNAPSHOT
-
-
- UTF-8
-
- /
- 3.1.1.RELEASE
- 3.1.0.RELEASE
-
-
-
- ${basedir}/src/main/java
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
- 1.7
- 1.7
-
-
-
- maven-assembly-plugin
- 2.5.1
-
-
- src/main/assembly/assembly.xml
-
-
-
-
-
-
-
-
- xerces
- xerces
- 2.4.0
-
-
- dom4j
- dom4j
- 1.6.1
-
-
- org.json
- json
- 20141113
-
-
-
- com.google.guava
- guava
- 14.0
-
-
-
- us.codecraft
- xsoup
- 0.2.4
-
-
-
- com.github.dreamhead
- moco-core
- 0.10.0
-
-
-
- org.slf4j
- slf4j-log4j12
- 1.7.7
-
-
-
-
-
- commons-collections
- commons-collections
- 3.2.1
-
-
-
- org.assertj
- assertj-core
- 1.7.0
-
-
-
- org.apache.commons
- commons-io
- 1.3.2
-
-
-
- com.jayway.jsonpath
- json-path
- 0.8.1
-
-
-
- com.alibaba
- fastjson
- 1.2.3
-
-
-
- org.jsoup
- jsoup
- 1.7.2
-
-
- org.apache.httpcomponents
- httpclient
- 4.3.4
-
-
- org.springframework
- spring-jdbc
- ${spring-version}
-
-
- org.apache.commons
- commons-lang3
- 3.1
-
-
- javax.servlet
- servlet-api
- 2.5
-
-
- mysql
- mysql-connector-java
- 5.1.18
-
-
- commons-dbcp
- commons-dbcp
- 1.3
-
-
- junit
- junit
- 4.7
- test
-
-
-
- org.mybatis
- mybatis
- 3.1.1
-
-
-
- org.mybatis
- mybatis-spring
- 1.1.1
-
-
-
- org.springframework
- spring-test
- ${spring-version}
- test
-
-
-
- net.trustie
- webmagic-core-fix
- 1.0
- system
- ${project.basedir}/lib/webmagic-core-fix.jar
-
-
-
- net.trustie
- webmagic-extension-fix
- 1.0
- system
- ${project.basedir}/lib/webmagic-extension-fix.jar
-
-
-
- org.seleniumhq.selenium
- selenium-java
- 2.42.2
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/51cto_blog.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/51cto_blog.xml
deleted file mode 100644
index 7d7c11471..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/51cto_blog.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 30
- div.rightbox>div.r_li>h4>a
- http://blog.51cto.com
- http://blog.51cto.com/original/0/
-
- 16
- 86400
- 0
-
-
-
-
- 51cto_blog
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/Copy of sample_site.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/Copy of sample_site.xml
deleted file mode 100644
index 77da3d5e0..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/Copy of sample_site.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-
-
-
- Spider
-
- csdn_topic
-
- 1
-
- 10
-
- div.content>div.list_1>ul>li>a
-
- http://bbs.csdn.net
-
- http://bbs.csdn.net/tech_hot_topics?page=
-
-
- 0
-
- 50
-
- 20000
-
- 10000
- 20000
-
- 1800000
- 7200000
-
- 3600000
- 7200000
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/apache.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/apache.xml
deleted file mode 100644
index cc7ba5c14..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/apache.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
-
-
-
-
- 1
- 1
- table>tbody>tr>td.body>div>div>ul>li> a
- http://projects.apache.org
- http://projects.apache.org/indexes/alpha.html
-
- 1
- 864000
- 0
-
-
-
-
- apache
- 10000
- 30000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml
deleted file mode 100644
index 42216357b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- 51cto_blog
-
- 1
-
- 1
-
- true
-
- 30
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://blog.51cto.com/original/0/1
-
-
-
- div.rightbox>div.r_li>h4>a
-
- 0
-
- 10
-
- 10000
-
- http://blog.51cto.com
-
- http://blog.51cto.com/original/0/
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 16
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of apache.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of apache.xml
deleted file mode 100644
index 8553844a8..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of apache.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- apache
-
- 1
-
- 1
-
- true
-
- 30
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://projects.apache.org/indexes/alpha.html
-
-
-
- table>tbody>tr>td.body>div>div>ul>li> a
-
- 0
-
- 10
-
- 10000
-
- http://projects.apache.org/indexes/alpha.html
-
- http://projects.apache.org/indexes/alpha.html
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 1
-
-
- 8640000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml
deleted file mode 100644
index 1b1e2c389..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- cnblogs_news
-
- 1
-
- 1
-
- true
-
- 4845
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://news.cnblogs.com/n/page/1
-
-
-
- div#news_list>div.news_block>div.content>h2.news_entry>a
-
- 0
-
- 10
-
- 10000
-
- http://news.cnblogs.com
-
- http://news.cnblogs.com/n/page/
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 5
-
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml
deleted file mode 100644
index 9319eb0ae..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- cnblogs_q_solved
-
- 1
-
- 1
-
- true
-
- 160
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 4000
-
- http://q.cnblogs.com/list/solved?page=1
-
-
-
- div.one_entity>div.news_item>h2.news_entry>a
-
- 0
-
- 10
-
- 10000
-
- http://q.cnblogs.com
-
- http://q.cnblogs.com/list/solved?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml
deleted file mode 100644
index 72768f9b8..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-
-
-
-
- Spider
-
- 1
-
- cnblogs_q_unsolved
-
- 1
-
- 1
-
- true
-
- 160
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 4000
-
- http://q.cnblogs.com/list/unsolved?page=1
-
-
-
- div.one_entity>div.news_item>h2.news_entry>a
-
- 0
-
- 10
-
- 10000
-
- http://q.cnblogs.com
-
- http://q.cnblogs.com/list/unsolved?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of code_project.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of code_project.xml
deleted file mode 100644
index 3bb9f53a9..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of code_project.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- codeproject
-
- 1
-
- 1
-
- true
-
- 21
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=1
-
-
-
- div.content-list-item>div.title>a
-
- 0
-
- 10
-
- 10000
-
- http://www.codeproject.com
-
- http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 1
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml
deleted file mode 100644
index 4eb3d847f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- dewen_question
-
- 1
-
- 1
-
- true
-
- 1114
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.dewen.io/questions?page=1
-
-
-
- div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a
-
- 0
-
- 10
-
- 10000
-
- http://www.dewen.io
-
- http://www.dewen.io/questions?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 1
-
-
-
- 432000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of gna.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of gna.xml
deleted file mode 100644
index f1bb2cef0..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of gna.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- gna
-
- 1
-
- 1
-
- true
-
- 1348
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=1&max_rows=1#results
-
-
-
- table>tbody>tr>td>a
-
- 0
-
- 10
-
- 10000
-
- http://gna.org
-
- http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=
-
- &max_rows=1#results
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 200
-
-
- 432000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml
deleted file mode 100644
index 0687a8e34..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- lupaworld
-
- 1
-
- 2
-
- true
-
- 479
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.lupaworld.com/forum-13746-2.html
-
-
-
- table#threadlisttableid>tbody>tr>th>a
-
- 0
-
- 10
-
- 10000
-
- http://www.lupaworld.com
-
- http://www.lupaworld.com/forum-13746-
-
- .html
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 2592000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of openhub.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of openhub.xml
deleted file mode 100644
index f48e5dc87..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of openhub.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-
-
-
- Spider
-
- 1
-
- openhub
-
- 1
-
- 1
-
- true
-
- 66784
-
- 5
-
- 60000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- https://www.openhub.net/p?page=1&q=&sort=users
-
-
-
- div.project>h2.title>a
-
- 0
-
- 10
-
- 60000
-
- https://www.openhub.net
-
- https://www.openhub.net/p?page=
-
- &q=&sort=users
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 500
-
- 100000
-
-
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of phpchina.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of phpchina.xml
deleted file mode 100644
index 712aa22d2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of phpchina.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- phpchina
-
- 1
-
- 1
-
- true
-
- 1289
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://bbs.phpchina.com/forum-17-1.html
-
-
-
- table#threadlisttableid>tbody>tr>th>a.s.xst
-
- 0
-
- 10
-
- 10000
-
- http://bbs.phpchina.com
-
- http://bbs.phpchina.com/forum-17-
-
- .html
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 1296000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of softpedia.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of softpedia.xml
deleted file mode 100644
index 9d9c0eb87..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of softpedia.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- softpedia
-
- 1
-
- 1
-
- true
-
- 30
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://linux.softpedia.com/index1.shtml
-
-
-
- div#sjmp>div>h4>a
-
- 0
-
- 10
-
- 10000
-
- http://linux.softpedia.com
-
- http://linux.softpedia.com/index
-
- .shtml
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 10
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml
deleted file mode 100644
index ba738497c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- sourceforge
-
- 1
-
- 1
-
- true
-
- 17621
-
- 5
-
- 60000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 10000
-
- http://sourceforge.net/directory/?page=1
-
-
-
- ul.projects>li>div.project_info>header>a
-
- 0
-
- 10
-
- 10000
-
- http://sourceforge.net
-
- http://sourceforge.net/directory/?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 500
-
- 250000
-
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml
deleted file mode 100644
index 21713db99..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- stackoverflow
-
- 1
-
- 1
-
- true
-
- 25000
-
- 5
-
- 60000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 10000
-
- http://stackoverflow.com/questions?pagesize=50&sort=newest&page=1
-
-
-
- div#mainbar>div#questions>div.question-summary>div.summary>h3>a
-
- 0
-
- 10
-
- 10000
-
- http://stackoverflow.com
-
- http://stackoverflow.com/questions?pagesize=50&sort=newest&page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 250
-
- 20000
-
- 2000
- 3000
-
-
- 1800000
- 2400000
-
-
- 7200000
- 14400000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml
deleted file mode 100644
index 0f13b714a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- linuxtone
-
- 1
-
- 1
-
- true
-
- 64
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://bbs.linuxtone.org/forum-15-1.html
-
-
-
- table>tbody>tr>th>a
-
- 0
-
- 10
-
- 10000
-
- http://bbs.linuxtone.org
-
- http://bbs.linuxtone.org/forum-15-
-
- .html
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 2592000
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml
deleted file mode 100644
index 8c9e1d8ad..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- iteye_blog
-
- 1
-
- 1
-
- true
-
- 15
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.iteye.com/blogs?page=1
-
-
-
- div.content>h3>a
-
- 0
-
- 10
-
- 10000
-
- http://www.iteye.com
-
- http://www.iteye.com/blogs?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 2
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/bytes.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/bytes.xml
deleted file mode 100644
index 45f49d0eb..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/bytes.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- question
- 1
- 2000
- table#threadslist>tbody>tr>td>div>div.threadbit_title>a
- http://bytes.com
- http://bytes.com/answers/
- /
- 2
- 86400
- 0
-
-
-
-
- bytes
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_news.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_news.xml
deleted file mode 100644
index 16565d865..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_news.xml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-
-
-
-
-
-
-
- 1
- 4845
- div#news_list>div.news_block>div.content>h2.news_entry>a
- http://news.cnblogs.com
- http://news.cnblogs.com/n/page/
-
- 5
- 86400
- 0
-
-
-
-
- cnblogs_news
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_q_solved.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_q_solved.xml
deleted file mode 100644
index 17598cd7c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_q_solved.xml
+++ /dev/null
@@ -1,27 +0,0 @@
-
-
-
-
-
- 1
- 160
- div.one_entity>div.news_item>h2.news_entry>a
- http://q.cnblogs.com
- http://q.cnblogs.com/list/solved?page=
-
- 2
- 86400
- 0
-
-
-
-
- cnblogs_q_solved
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml
deleted file mode 100644
index 1676a68a0..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 30
- div.one_entity>div.news_item>h2.news_entry>a
- http://q.cnblogs.com
- http://q.cnblogs.com/list/unsolved?page=
-
- 2
- 86400
- 0
-
-
-
-
- cnblogs_q_unsolved
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/code_project.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/code_project.xml
deleted file mode 100644
index 580834ed4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/code_project.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- 1
- 259
- div.content-list-item>div.title>a
- http://www.codeproject.com
-
- http://www.codeproject.com/script/Articles/Latest.aspx?la_as=366&la_app=20&la_minscore=1.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=
-
- 1
- 86400
- 0
-
-
-
-
- codeproject
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_ask.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_ask.xml
deleted file mode 100644
index f45d7a50e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_ask.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 2496
- div.common_con>div.questions_detail_con>dl>dt>a
- http://ask.csdn.net
- http://ask.csdn.net/?page=
-
- 25
- 86400
- 0
-
-
-
-
- csdn_ask
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_blog.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_blog.xml
deleted file mode 100644
index b00479da0..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_blog.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 101
- div.main_center>div.blog_list>h1>a:not(.category)
- http://blog.csdn.net
- http://blog.csdn.net/index.html?&page=
-
- 25
- 86400
- 0
-
-
-
-
- csdn_blog
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_topic.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_topic.xml
deleted file mode 100644
index 1749acd3d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/csdn_topic.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-
-
-
-
-
- mobile
- 1
- 808
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/Mobile?page=
-
- 8
- 86400
- 0
-
-
-
- CloudComputing
- 1
- 22
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/CloudComputing?page=
-
- 1
- 86400
- 0
-
-
-
- java
- 1
- 2555
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/Java?page=
-
- 4
- 86400
- 0
-
-
-
- web
- 1
- 1853
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/WebDevelop?page=
-
- 3
- 86400
- 0
-
-
-
- OtherLanguage
- 1
- 336
- div.wrap > div.content > table > tbody > tr> td.title > a
- http://bbs.csdn.net
- http://bbs.csdn.net/forums/OtherLanguage?page=
-
- 1
- 86400
- 0
-
-
-
-
- csdn_topic
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/dewen_question.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/dewen_question.xml
deleted file mode 100644
index 6038c0293..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/dewen_question.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
-
-
-
-
- 1
- 1114
- div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a
- http://www.dewen.io
- http://www.dewen.io/questions?page=
-
- 1
- 432000
- 0
-
-
-
-
- dewen_question
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/freecode.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/freecode.xml
deleted file mode 100644
index 943e6ded2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/freecode.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
- Spider
-
- 5
-
- freecode
-
- 5
-
- 1
-
- true
-
- 4799
-
- 5
-
- 30000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 10000
-
- http://www.freecode.com/projects?page=1
-
-
-
- div.project>h2.release-head>a
-
- 0
-
- 10
-
- 60000
-
- http://www.freecode.com
-
- http://www.freecode.com/projects?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 5
-
- 20000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/gna.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/gna.xml
deleted file mode 100644
index 712713f46..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/gna.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 1348
- table>tbody>tr>td>a
- http://gna.org
- http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=
- &max_rows=1#results
- 100
- 86400
- 0
-
-
-
-
- gna
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/ibm_post.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/ibm_post.xml
deleted file mode 100644
index 4d55bc4fd..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/ibm_post.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- blog
- 1
- 138
- table>tbody>tr>td>a
- http://www.ibm.com
- http://www.ibm.com/developerworks/cn/views/opensource/libraryview.jsp?site_id=10&contentarea_by=%E6%89%80%E6%9C%89%E4%B8%93%E5%8C%BA&sort_by=&sort_order=2&start=
- 00&end=13874&topic_by=-1&product_by=&type_by=%E6%89%80%E6%9C%89%E7%B1%BB%E5%88%AB&show_abstract=false&search_by=&industry_by=&series_title_by=
- 1
- 6048000
- 0
-
-
-
-
- ibm_post
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/ibm_project.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/ibm_project.xml
deleted file mode 100644
index 3e0e2be41..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/ibm_project.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 1
- table>tbody>tr>td>a
- http://www.ibm.com
- http://www.ibm.com/developerworks/views/opensource/projects.jsp
-
- 1
- 864000
- 0
-
-
-
-
- ibm_project
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_ask.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_ask.xml
deleted file mode 100644
index 72778075a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_ask.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
- Spider
-
- 5
-
- iteye_ask
-
- 5
-
- 1
-
- true
-
- 5329
-
- 5
-
- 30000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://www.iteye.com/ask?page=1
-
-
-
- div#ask_list>div.question-summary>div.summary>h3>a
-
- 0
-
- 10
-
- 60000
-
- http://www.iteye.com/
-
- http://www.iteye.com/ask?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 50
-
- 20000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_bbs.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_bbs.xml
deleted file mode 100644
index 6ca237d4b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_bbs.xml
+++ /dev/null
@@ -1,78 +0,0 @@
-
-
-
-
- mobile
- 1
- 111
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/mobile?page=
-
- 16
- 1296000
- 0
-
-
-
- language
- 1
- 242
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/language?page=
-
- 1
- 1296000
- 0
-
-
-
- web
- 1
- 320
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/web?page=
-
- 1
- 1296000
- 0
-
-
- java
- 1
- 754
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/Java?page=
-
- 1
- 1296000
- 0
-
-
- tech
- 1
- 242
- table#forum_main>tbody>tr>td>a
- http://www.iteye.com
- http://www.iteye.com/forums/board/Tech?page=
-
- 1
- 1296000
- 0
-
-
-
-
- iteye_bbs
- 30000
- 50000
- 3600000
- 7200000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_blog.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_blog.xml
deleted file mode 100644
index b45fa450e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/iteye_blog.xml
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
- blog
- 1
- 15
- div.content>h3>a
- http://www.iteye.com
- http://www.iteye.com/blogs?page=
-
- 2
- 86400
- 0
-
-
-
-
- iteye_blog
- 30000
- 50000
- 3600000
- 7200000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/javaforge.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/javaforge.xml
deleted file mode 100644
index 0bdc05cf9..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/javaforge.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
- Spider
-
- 1
-
- javaforge
-
- 1
-
- 1
-
- true
-
- 31
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 5000
-
- http://javaforge.com/listProjects.spr?page=1&displayAllProjects=on
-
-
-
- table#entry>tbody>tr>td:nth-child(1) > a
-
- 0
-
- 10
-
- 10000
-
- http://javaforge.com
-
- http://javaforge.com/listProjects.spr?page=
-
- &displayAllProjects=on
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 10
-
-
- 86400
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/lagou.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/lagou.xml
deleted file mode 100644
index 4bfade221..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/lagou.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
- houduan
- 1
- 30
- #container > div.content>ul> li> div> div > a
- http://www.lagou.com
- http://www.lagou.com/jobs/list_%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91?kd=%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91&spc=1&pl=&gj=&xl=&yx=&gx=&st=&labelWords=label&lc=&workAddress=&city=%E5%85%A8%E5%9B%BD&requestId=&pn=
-
- 16
- 86400
- 0
-
-
-
-
- lagou
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/linuxtone.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/linuxtone.xml
deleted file mode 100644
index 1930f0fa4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/linuxtone.xml
+++ /dev/null
@@ -1,81 +0,0 @@
-
-
-
-
-
- System
- 1
- 64
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-15-
- .html
- 1
- 2592000
- 0
-
-
-
- server
- 1
- 30
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-22-
- .html
- 1
- 2592000
- 0
-
-
-
- dbserver
- 1
- 32
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-24-
- .html
- 1
- 2592000
- 0
-
-
-
- yngz
- 1
- 32
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-49-
- .html
- 1
- 2592000
- 0
-
-
-
- fzjh
- 1
- 24
- table>tbody>tr>th>a
- http://bbs.linuxtone.org
- http://bbs.linuxtone.org/forum-26-
- .html
- 1
- 2592000
- 0
-
-
-
-
- linuxtone
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/lupaworld.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/lupaworld.xml
deleted file mode 100644
index 57d3e0bc2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/lupaworld.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
-
-
- 1
- 479
- table#threadlisttableid>tbody>tr>th>a
- http://www.lupaworld.com
- http://www.lupaworld.com/forum-13746-
- .html
- 2
- 2592000
- 0
-
-
-
-
- lupaworld
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/neitui.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/neitui.xml
deleted file mode 100644
index 7854326cb..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/neitui.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 610
- ul>li>div.cont>div.jobnote.clearfix > div.jobnote-l>a
- http://www.neitui.me
- http://www.neitui.me/neitui/type=all&page=
- .html
- 16
- 86400
- 0
-
-
-
-
- neitui
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/openhub.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/openhub.xml
deleted file mode 100644
index 72fb3e3e6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/openhub.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-
-
-
-
-
-
- 1
- 66784
- div.project>h2.title>a
- https://www.openhub.net
-
-
- https://www.openhub.net/p?page=
- &q=&sort=new
- 8
- 86400
-
- 0
-
-
-
-
- openhub
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/oschina_project.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/oschina_project.xml
deleted file mode 100644
index 0cc47a98f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/oschina_project.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
-
-
-
-
- 1
- 1702
- div.ProjectList>ul.List>li>h3>a
- http://www.oschina.net
- http://www.oschina.net/project/list?prefix=&sort=time&p=
-
- 1
- 86400
- 0
-
-
-
-
- oschina_project
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/oschina_question.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/oschina_question.xml
deleted file mode 100644
index d059d37c4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/oschina_question.xml
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
- 1
- 2361
- ul.list>li.question>div.question-detail>strong>a
- http://www.oschina.net
- http://www.oschina.net/question?catalog=1&show=active&p=
-
- 1
- 86400
- 0
-
-
-
-
- oschina_question
- 30000
- 60000
- 7200000
- 10800000
- 1800000
- 3600000
-
-
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/ossean.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/ossean.xml
deleted file mode 100644
index f35470b83..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/ossean.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-
-
-
- Spider
-
- 3
-
- ossean
-
- 3
-
- 1
-
- true
-
- 5777
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 500
-
- http://localhost/open_source_projects?page=1
-
-
-
- ul.projects>li.project-table>div.root>a
-
- 0
-
- 10
-
- 10000
-
- http://localhost
-
- http://localhost/open_source_projects?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 5
-
- 0
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/phpchina.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/phpchina.xml
deleted file mode 100644
index ef48a9514..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/phpchina.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-
-
-
-
-
-
-
- ask
- 1
- 1289
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-17-
- .html
- 2
- 1296000
- 0
-
-
-
- server_os
- 1
- 91
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-195-
- .html
- 2
- 1296000
- 0
-
-
-
- web
- 1
- 81
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-213-
- .html
- 2
- 1296000
- 0
-
-
-
- db
- 1
- 43
- table#threadlisttableid>tbody>tr>th>a.s.xst
- http://bbs.phpchina.com
- http://bbs.phpchina.com/forum-196-
- .html
- 2
- 1296000
- 0
-
-
-
-
- phpchina
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/sample_site.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/sample_site.xml
deleted file mode 100644
index f8224e89d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/sample_site.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-
-
-
- Spider
-
- 3
-
- csdn_topic
-
- 3
-
- 1
-
- true
-
- 10
-
- 5
-
- 10000
-
- 0
-
- 5
-
- Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)
-
- 4000
-
- http://bbs.csdn.net/tech_hot_topics?page=1
-
-
-
- div.content>div.list_1>ul>li>a
-
- 0
-
- 10
-
- 10000
-
- http://bbs.csdn.net
-
- http://bbs.csdn.net/tech_hot_topics?page=
-
-
-
- CSSPath
-
- 192.168.245.1
-
- 3128
-
- 0
-
- 50
-
- 20000
-
-
- 10000
- 20000
-
-
- 1800000
- 7200000
-
-
- 3600000
- 7200000
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/slashdot.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/slashdot.xml
deleted file mode 100644
index b85f0ed5e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/slashdot.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-
-
-
-
-
-
- 1
- 66
-<<<<<<< HEAD
- #firehoselist>article>header>h2>span>a
- http://slashdot.org
- http://slashdot.org/?page=
-
- 2
-=======
-
- #firehoselist>article>header>h2>span>a
- http://slashdot.org
- http://slashdot.org/?page=
- .html
- 2
->>>>>>> tmp
- 86400
- 0
-
-
-
-
- slashdot
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-<<<<<<< HEAD
-
-=======
-
->>>>>>> tmp
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/softpedia.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/softpedia.xml
deleted file mode 100644
index 5ace82f8c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/softpedia.xml
+++ /dev/null
@@ -1,134 +0,0 @@
-
-
-
-
-
- linux
- 1
- 30
- h4>a
- http://linux.softpedia.com
- http://linux.softpedia.com/index
- .shtml
- 3
- 86400
- 0
-
-
-
- win
- 1
- 30
- h4>a
- http://win.softpedia.com
- http://win.softpedia.com/index
- .shtml
- 3
- 86400
- 0
-
-
-
- mac
- 1
- 35
- h4>a
- http://mac.softpedia.com
- http://mac.softpedia.com/index
- .shtml
- 5
- 86400
- 0
-
-
-
- windows-phone
- 1
- 46
- h4>a
- http://mobile.softpedia.com
- http://mobile.softpedia.com/windows-phone,
-
- 5
- 86400
- 0
-
-
-
- android
- 1
- 35
- h4>a
- http://mobile.softpedia.com
- http://mobile.softpedia.com/android,
-
- 1
- 86400
- 0
-
-
-
- ios
- 1
- 22
- h4>a
- http://mobile.softpedia.com
- http://mobile.softpedia.com/ios,
-
- 1
- 86400
- 0
-
-
-
- driver
- 1
- 30
- h4>a
- http://drivers.softpedia.com
- http://drivers.softpedia.com/index
- .shtml
- 10
- 86400
- 0
-
-
-
- game
- 1
- 30
- h4>a
- http://games.softpedia.com
- http://games.softpedia.com/index
- .shtml
- 2
- 86400
- 0
-
-
-
- webscripts
- 1
- 30
- h4>a
- http://webscripts.softpedia.com
- http://webscripts.softpedia.com/index/latest-scripts-3
- -0-0.html
- 3
- 86400
- 0
-
-
-
-
-
- softpedia
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/sourceforge.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/sourceforge.xml
deleted file mode 100644
index 9bc36d8e8..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/sourceforge.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-
-
-
-
-
-
- 1
- 17621
- ul.projects>li>div.project_info>header>a
- http://sourceforge.net
-<<<<<<< HEAD
-
-=======
-
->>>>>>> tmp
- http://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update&page=
-
-
-
- 8
- 86400
- 0
-
-
-
-
- sourceforge
- 10000
- 20000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/spider_test.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/spider_test.xml
deleted file mode 100644
index 7a880b14b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/spider_test.xml
+++ /dev/null
@@ -1,66 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- database
- 1
- 20
- div#news_list>div.news_block>div.content>h2.news_entry>a
- http://localhost:8080/SpiderTest
- http://localhost:8080/SpiderTest/database?page=
-
- 15
- 300
- 0
-
-
- os
- 1
- 20
- div#news_list>div.news_block>div.content>h2.news_entry>a
- http://localhost:8080/SpiderTest
- http://localhost:8080/SpiderTest/os?page=
-
- 15
- 300
- 0
-
-
-
-
- SpiderTes
- 1000
- 3000
- 1800000
- 7200000
- 3600000
- 7200000
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sites/stackoverflow.xml b/crawler/moreSmarterCrawler/fetch_networks/sites/stackoverflow.xml
deleted file mode 100644
index 82262eda4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sites/stackoverflow.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-
-
-
-
-
- 1
- 25000
- div#mainbar>div#questions>div.question-summary>div.summary>h3>a
- http://stackoverflow.com
- http://stackoverflow.com/questions?pagesize=50&sort=newest&page=
-
- 8
-
- 3600
- 0
-
-
-
-
- stackoverflow
- 1000
- 2000
- 300000
- 600000
- 7200000
- 14400000
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/spider.conf.xml b/crawler/moreSmarterCrawler/fetch_networks/spider.conf.xml
deleted file mode 100644
index 4c4103350..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/spider.conf.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-
-
-
-Spider
-1
-ques_oschina
-2
-1
-true
-2
-5
-10000
-5000
-5
-Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36
-10000
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sql/detail_html.sql b/crawler/moreSmarterCrawler/fetch_networks/sql/detail_html.sql
deleted file mode 100644
index 412202765..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sql/detail_html.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
-`crawledTime` datetime NULL DEFAULT NULL ,
-`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=1
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sql/error_page.sql b/crawler/moreSmarterCrawler/fetch_networks/sql/error_page.sql
deleted file mode 100644
index 98585115e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sql/error_page.sql
+++ /dev/null
@@ -1,15 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
-`crawledTime` datetime NULL DEFAULT NULL ,
-`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=8388
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sql/list_html.sql b/crawler/moreSmarterCrawler/fetch_networks/sql/list_html.sql
deleted file mode 100644
index 1ea3a0622..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sql/list_html.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
-`crawledTime` datetime NULL DEFAULT NULL ,
-`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`extracted` bigint(20) NULL DEFAULT NULL ,
-PRIMARY KEY (`id`),
-INDEX `pagemd5` (`pageMd5`) USING BTREE
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=25
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sql/pointers.sql b/crawler/moreSmarterCrawler/fetch_networks/sql/pointers.sql
deleted file mode 100644
index e8185e406..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sql/pointers.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`table_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`pointer` int(11) NULL DEFAULT NULL ,
-`created_at` datetime NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=3
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/sql/url.sql b/crawler/moreSmarterCrawler/fetch_networks/sql/url.sql
deleted file mode 100644
index 626ba4b67..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/sql/url.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-(
-`id` int(11) NOT NULL AUTO_INCREMENT ,
-`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
-`timestamp` bigint(20) NULL DEFAULT NULL ,
-`extractedTime` datetime NULL DEFAULT NULL ,
-PRIMARY KEY (`id`)
-)
-ENGINE=InnoDB
-DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
-AUTO_INCREMENT=1
-ROW_FORMAT=COMPACT
-;
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/assembly/assembly.xml b/crawler/moreSmarterCrawler/fetch_networks/src/main/assembly/assembly.xml
deleted file mode 100644
index baf8ba912..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/assembly/assembly.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
- jar-with-dependencies-without-resources
-
- dir
-
- false
-
-
- /
- false
- false
- runtime
-
-
- /
- false
- false
- system
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/META-INF/MANIFEST.MF b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/META-INF/MANIFEST.MF
deleted file mode 100644
index c34e50601..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/META-INF/MANIFEST.MF
+++ /dev/null
@@ -1,3 +0,0 @@
-Manifest-Version: 1.0
-Main-Class: net.trustie.webmagic.crawler.OSChinaQuestionCrawler
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/CSSPathExtractor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/CSSPathExtractor.java
deleted file mode 100644
index f7201cb3a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/CSSPathExtractor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.selector.Html;
-
-public class CSSPathExtractor implements Extractor{
- private String path = "";
-
- /**
- * @param path
- */
- public CSSPathExtractor(String path) {
- this.path = path;
- }
-
- @Override
- public Set extract(Html html) {
- // TODO Auto-generated method stub
- Document doc = html.getDocument();
- Elements eles = doc.select(path);
- Set urlSet = new HashSet();
- for(Element e :eles){
- //针对gnaurl以两个点开头
- String url = e.attr("href");
- if(url.startsWith("http://gna.org/..")){
- int pos = url.indexOf("..");
- url = url.substring(0,pos) + url.substring(pos+3,url.length());
- }
-
- urlSet.add(url);
- }
-
- return urlSet;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/Extractor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/Extractor.java
deleted file mode 100644
index 242448ae4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/Extractor.java
+++ /dev/null
@@ -1,9 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.Set;
-
-import us.codecraft.webmagic.selector.Html;
-
-public interface Extractor {
- public Set extract(Html html);
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/URLPatternExtractor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/URLPatternExtractor.java
deleted file mode 100644
index 24908cdbf..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/URLPatternExtractor.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import net.trustie.webmagic.utils.StringUtils;
-import us.codecraft.webmagic.selector.Html;
-
-public class URLPatternExtractor implements Extractor{
- private String pattern = "";
-
-
- /**
- * @param pattern
- */
- public URLPatternExtractor(String pattern) {
- this.pattern = pattern;
- }
-
-
- @Override
- public Set extract(Html html) {
- // TODO Auto-generated method stub
- List postUrls = html.links().regex(pattern).all();
-
- return StringUtils.removeDuplicate(postUrls);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/XPathExtractor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/XPathExtractor.java
deleted file mode 100644
index 7fb7a9f1c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/extrator/XPathExtractor.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package net.trustie.webmagic.extrator;
-
-import java.util.List;
-import java.util.Set;
-
-import net.trustie.webmagic.utils.StringUtils;
-import us.codecraft.webmagic.selector.Html;
-
-public class XPathExtractor implements Extractor {
- private String path = "";
-
- /**
- * @param path
- */
- public XPathExtractor(String path) {
- this.path = path;
- }
-
- @Override
- public Set extract(Html html) {
- // TODO Auto-generated method stub
- List urlList = html.xpath(path).all();
- return StringUtils.removeDuplicate(urlList);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Configure.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Configure.java
deleted file mode 100644
index a3b71587e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/Configure.java
+++ /dev/null
@@ -1,600 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.utils.SubSite;
-
-import org.apache.log4j.Logger;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.InvalidPropertiesFormatException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Queue;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-
-public class Configure {
- private int sleepTimeThread = 5000;
-
- //--长时间睡眠时候的睡眠时间的上下限 -->
- private int minlongSleepTime = 1800000;
- private int maxlongSleepTime = 7200000;
-
- //长时间睡眠时候的睡眠间隔的上下限 -->
- private int minlongSleepInterval = 3600000;
- private int maxlongSleepInterval = 7200000;
- //用以设置请求间隔的两个阈值以产生随机数 (毫秒)
- private int minInterval = 10000;
- private int maxInterval = 20000;
-
- private int retryTimes = 0;
- private int cycleRetryTimes = 0;
- private int timeOut = 5000;
- private String userAgent = "UserAgent.Browser";
- private String charset;
- private int threadNum = 5;
- private boolean isSpawnUrl = true;
- private int startPageIndex = 1;
- private int endInPageIndex = 6;
- private int pageF = 1;
- private int sleepTimeSpider = 36000;
- private String domain;
- private String startUrl;
- private String urlList;
- private String urlPost;
- private int offset;
- private int limitLine;
- private int noUrlWaitTime;
- private String fixAllRelativeHrefs;
- private Properties prop = new Properties();
- private String prefix;
- private String postfix;
- private String extractMethod;
- private String proxyIp;
- private String proxyPort;
- private int mode;
- private int incrementalPages;
- private int incrementSleepTime;
- //private List subSites = new LinkedList();
- private Queue subSites = new LinkedList();
- Logger log4j = Logger.getLogger(Configure.class);
- Document doc = null;
- public Configure() {
- try {
- FileInputStream fis = null;
- fis = new FileInputStream("spider.conf.xml");
- prop.loadFromXML(fis);
- prop.list(System.out);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
-
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- public static void main(String args[]){
- Configure conf = new Configure("spider_test");
- }
- public Configure(String confPath) {
- /*confPath = confPath + ".xml";
- try {
- URL url = ClassLoader.getSystemResource(confPath);
- InputStream is = url.openStream();
- prop.loadFromXML(is);
- // prop.list(System.out);
- } catch (FileNotFoundException e) {
- log4j.error("FileNotFoundException & loading default config xml! & no default config file");
- e.printStackTrace();
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();*/
- confPath = confPath + ".xml";
- URL url = ClassLoader.getSystemResource(confPath);
- InputStream is;
- try {
- is = url.openStream();
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder db = dbf.newDocumentBuilder();
- doc = db.parse(is);
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (ParserConfigurationException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (SAXException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- config();
- }
-
-
- private void config() {
-
- //star modify----------------------------
- Element root = doc.getDocumentElement();
- //获取站点
- NodeList sites = ((Element)root.getElementsByTagName("subSites").item(0)).getElementsByTagName("subSite");
- NodeList properties = ((Element)root.getElementsByTagName("properties").item(0)).getChildNodes();
- for(int i =0; i < sites.getLength(); i++){
- //System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
- Node subSite= sites.item(i);
- addSubSite(subSite,subSites);
- }
- //用map存储properties ,然后再赋值;
- Map propertiesMap = new HashMap();
- for(int i =0; i < properties.getLength(); i++){
- //System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
- propertiesMap.put(properties.item(i).getNodeName(), properties.item(i).getTextContent());
- }
- this.domain = propertiesMap.get("domain");
- this.minInterval = Integer.parseInt(propertiesMap.get("minInterval")) ;
- this.maxInterval = Integer.parseInt(propertiesMap.get("maxInterval")) ;
-
- this.minlongSleepTime = Integer.parseInt(propertiesMap.get("minlongSleepTime")) ;
- this.maxlongSleepTime = Integer.parseInt(propertiesMap.get("maxlongSleepTime")) ;
-
- this.minlongSleepInterval = Integer.parseInt(propertiesMap.get("minlongSleepInterval")) ;
- this.maxlongSleepInterval= Integer.parseInt(propertiesMap.get("maxlongSleepInterval")) ;
- //System.out.println(propertiesMap); this.retryTimes = 5;
- this.timeOut = 20000;
- this.sleepTimeThread = 0;
- this.cycleRetryTimes = 5;
- this.pageF = 1;
- this.threadNum = 1;
- this.isSpawnUrl = true;
- this.offset = 0;
- this.limitLine = 10;
- this.noUrlWaitTime = 10000;
- this.extractMethod = "CSSPath";
- this.proxyIp = "192.168.245.1";
- this.proxyPort = "3128";
- this.userAgent = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
-
- //目前url修正和信息页抽取共用一个模板
- this.urlPost = this.subSites.peek().getUrlPost();
- this.fixAllRelativeHrefs = this.subSites.peek().getFixAllRelativeHrefs();
- //end of star modify-------------------------
-
- /*this.sleepTimeThread = Integer.parseInt(prop
- .getProperty("sleepTimeThread"));
-
-
- this.timeOut = Integer.parseInt(prop.getProperty("timeOut"));
- this.retryTimes = Integer.parseInt(prop.getProperty("retryTimes"));
- this.cycleRetryTimes = Integer.parseInt(prop
- .getProperty("cycleRetryTimes"));
- this.threadNum = Integer.parseInt(prop.getProperty("threadNum"));
- this.startPageIndex = Integer.parseInt(prop
- .getProperty("startPageIndex"));
- this.endInPageIndex = Integer.parseInt(prop
- .getProperty("endInPageIndex"));
- this.pageF = Integer.parseInt(prop.getProperty("pageF"));
- this.sleepTimeSpider = Integer.parseInt(prop
- .getProperty("sleepTimeSpider"));
- this.isSpawnUrl = Boolean.parseBoolean(prop.getProperty("isSpawnUrl"));
- this.userAgent = prop.getProperty("userAgent");
- this.domain = prop.getProperty("domain");
- this.startUrl = prop.getProperty("startUrl");
- this.urlList = prop.getProperty("urlList");
- this.urlPost = prop.getProperty("urlPost");
- this.offset = Integer.parseInt(prop.getProperty("offset"));
- this.limitLine = Integer.parseInt(prop.getProperty("limitLine"));
- this.noUrlWaitTime = Integer
- .parseInt(prop.getProperty("noUrlWaitTime"));
-
- this.fixAllRelativeHrefs = prop.getProperty("fixAllRelativeHrefs");
- this.prefix = prop.getProperty("prefixUrl");
- this.postfix = prop.getProperty("postfixUrl");
- this.extractMethod = prop.getProperty("extractMethod");
- this.proxyIp = prop.getProperty("proxyIp");
- this.proxyPort =prop.getProperty("proxyPort");
- this.mode = Integer.parseInt(prop.getProperty("mode"));
- this.incrementalPages = Integer.parseInt(prop.getProperty("incrementalPages"));
- this.incrementSleepTime = Integer.parseInt(prop.getProperty("incrementSleepTime"));
-
- this.minInterval = Integer.parseInt(prop.getProperty("minInterval"));
- this.maxInterval = Integer.parseInt(prop.getProperty("maxInterval"));*/
- }
-
-
-
- private void addSubSite(Node site, Queue subSites2) {
- NodeList sitePros = site.getChildNodes();
- Map siteProMap = new HashMap();
- for(int i =0; i < sitePros.getLength(); i++){
- //System.out.println(nods.item(i).getNodeName() + ":"+nods.item(i).getTextContent());
- siteProMap.put(sitePros.item(i).getNodeName(), sitePros.item(i).getTextContent());
- }
-
- //System.out.println(siteProMap);
- SubSite subSite = new SubSite();
-
- subSite.setName(siteProMap.get("name"));
- subSite.setStartPageIndex(Integer.parseInt(siteProMap.get("startPageIndex")));
- subSite.setEndInPageIndex(Integer.parseInt(siteProMap.get("endInPageIndex")));
- subSite.setFixAllRelativeHrefs(siteProMap.get("fixAllRelativeHrefs"));
- subSite.setPrefixUrl(siteProMap.get("prefixUrl"));
- subSite.setPostfixUrl(siteProMap.get("postfixUrl"));
- subSite.setUrlPost(siteProMap.get("urlPost"));
- subSite.setIncrementalPages(Integer.parseInt(siteProMap.get("incrementalPages")));
- subSite.setIncrementSleepTime(Integer.parseInt(siteProMap.get("incrementSleepTime")));
-
- subSites2.offer(subSite);
- //System.out.println(subSite.getStartPageIndex());
- }
-
- /**
- * @return the incrementSleepTime
- */
- public int getIncrementSleepTime() {
- return incrementSleepTime;
- }
-
- /**
- * @param incrementSleepTime the incrementSleepTime to set
- */
- public void setIncrementSleepTime(int incrementSleepTime) {
- this.incrementSleepTime = incrementSleepTime;
- }
-
- public int getSleepTimeThread() {
- return sleepTimeThread;
- }
-
- public void setSleepTimeThread(int sleepTimeThread) {
- this.sleepTimeThread = sleepTimeThread;
- }
-
- public int getRetryTimes() {
- return retryTimes;
- }
-
- public void setRetryTimes(int retryTimes) {
- this.retryTimes = retryTimes;
- }
-
- public int getCycleRetryTimes() {
- return cycleRetryTimes;
- }
-
- public void setCycleRetryTimes(int cycleRetryTimes) {
- this.cycleRetryTimes = cycleRetryTimes;
- }
-
- public int getTimeOut() {
- return timeOut;
- }
-
- public void setTimeOut(int timeOut) {
- this.timeOut = timeOut;
- }
-
- public String getUserAgent() {
- return userAgent;
- }
-
- public void setUserAgent(String userAgent) {
- this.userAgent = userAgent;
- }
-
- public String getCharset() {
- return charset;
- }
-
- public void setCharset(String charset) {
- this.charset = charset;
- }
-
- public int getThreadNum() {
- return threadNum;
- }
-
- public void setThreadNum(int threadNum) {
- this.threadNum = threadNum;
- }
-
- public boolean isSpawnUrl() {
- return isSpawnUrl;
- }
-
- public void setSpawnUrl(boolean isSpawnUrl) {
- this.isSpawnUrl = isSpawnUrl;
- }
-
- public int getStartPageIndex() {
- return startPageIndex;
- }
-
- public void setStartPageIndex(int startPageIndex) {
- this.startPageIndex = startPageIndex;
- }
-
- public int getEndInPageIndex() {
- return endInPageIndex;
- }
-
- public void setEndInPageIndex(int endInPageIndex) {
- this.endInPageIndex = endInPageIndex;
- }
-
- public int getPageF() {
- return pageF;
- }
-
- public void setPageF(int pageF) {
- this.pageF = pageF;
- }
-
- public int getSleepTimeSpider() {
- return sleepTimeSpider;
- }
-
- public void setSleepTimeSpider(int sleepTimeSpider) {
- this.sleepTimeSpider = sleepTimeSpider;
- }
-
- public String getDomain() {
- return domain;
- }
-
- public void setDomain(String domain) {
- this.domain = domain;
- }
-
- public String getStartUrl() {
- return startUrl;
- }
-
- public void setStartUrl(String startUrl) {
- this.startUrl = startUrl;
- }
-
- public String getUrlList() {
- return urlList;
- }
-
- public void setUrlList(String urlList) {
- this.urlList = urlList;
- }
-
- public String getUrlPost() {
- return urlPost;
- }
-
- public void setUrlPost(String urlPost) {
- this.urlPost = urlPost;
- }
-
- public Properties getProp() {
- return prop;
- }
-
- public void setProp(Properties prop) {
- this.prop = prop;
- }
-
- public Logger getLog4j() {
- return log4j;
- }
-
- public void setLog4j(Logger log4j) {
- this.log4j = log4j;
- }
-
- public int getOffset() {
- return offset;
- }
-
- public void setOffset(int offset) {
- this.offset = offset;
- }
-
- public int getLimitLine() {
- return limitLine;
- }
-
- public void setLimitLine(int limitLine) {
- this.limitLine = limitLine;
- }
-
- public int getNoUrlWaitTime() {
- return noUrlWaitTime;
- }
-
- public void setNoUrlWaitTime(int noUrlWaitTime) {
- this.noUrlWaitTime = noUrlWaitTime;
- }
-
-
-
-
- public String getFixAllRelativeHrefs() {
- return fixAllRelativeHrefs;
- }
-
- public void setFixAllRelativeHrefs(String fixAllRelativeHrefs) {
- this.fixAllRelativeHrefs = fixAllRelativeHrefs;
- }
-
- public String getPrefix() {
- return prefix;
- }
-
- public void setPrefix(String prefix) {
- this.prefix = prefix;
- }
-
- public String getPostfix() {
- return postfix;
- }
-
- public void setPostfix(String postfix) {
- this.postfix = postfix;
- }
-
- public String getExtractMethod() {
- return extractMethod;
- }
-
- public void setExtractMethod(String extraMethod) {
- this.extractMethod = extraMethod;
- }
-
-
- /**
- * @return the proxyIp
- */
- public String getProxyIp() {
- return proxyIp;
- }
-
- /**
- * @return the proxyPort
- */
- public String getProxyPort() {
- return proxyPort;
- }
-
- /**
- * @param proxyIp the proxyIp to set
- */
- public void setProxyIp(String proxyIp) {
- this.proxyIp = proxyIp;
- }
-
- /**
- * @param proxyPort the proxyPort to set
- */
- public void setProxyPort(String proxyPort) {
- this.proxyPort = proxyPort;
- }
-
- /**
- * @return the mode
- */
- public int getMode() {
- return mode;
- }
-
- /**
- * @param mode the mode to set
- */
- public void setMode(int mode) {
- this.mode = mode;
- }
-
- /**
- * @return the incrementalPages
- */
- public int getIncrementalPages() {
- return incrementalPages;
- }
-
- /**
- * @param incrementalPages the incrementalPages to set
- */
- public void setIncrementalPages(int incrementalPages) {
- this.incrementalPages = incrementalPages;
- }
-
-
-
- //每次请求的时间间隔
- public int getMinInterval() {
- return minInterval;
- }
-
- public void setMinInterval(int minInterval) {
- this.minInterval = minInterval;
- }
-
- public int getMaxInterval() {
- return maxInterval;
- }
-
- public void setMaxInterval(int maxInterval) {
- this.maxInterval = maxInterval;
- }
-
- public int getMinlongSleepTime() {
- return minlongSleepTime;
- }
-
- public void setMinlongSleepTime(int minlongSleepTime) {
- this.minlongSleepTime = minlongSleepTime;
- }
-
- public int getMaxlongSleepTime() {
- return maxlongSleepTime;
- }
-
- public void setMaxlongSleepTime(int maxlongSleepTime) {
- this.maxlongSleepTime = maxlongSleepTime;
- }
-
- public int getMinlongSleepInterval() {
- return minlongSleepInterval;
- }
-
- public void setMinlongSleepInterval(int minlongSleepInterval) {
- this.minlongSleepInterval = minlongSleepInterval;
- }
-
- public int getMaxlongSleepInterval() {
- return maxlongSleepInterval;
- }
-
- public void setMaxlongSleepInterval(int maxlongSleepInterval) {
- this.maxlongSleepInterval = maxlongSleepInterval;
- }
-
- public int getSubSiteNum() {
- return this.subSites.size();
- }
-
-
- // 获取下一个要处理的站点,并更新当前状态
- public SubSite nexSite() {
- SubSite subSite = this.subSites.poll();
- this.subSites.offer( subSite);
-
- this.startPageIndex = subSite.getStartPageIndex();
- this.endInPageIndex = subSite.getEndInPageIndex();
- this.fixAllRelativeHrefs = subSite.getFixAllRelativeHrefs();
- this.prefix = subSite.getPrefixUrl();
- this.postfix = subSite.getPostfixUrl();
- this.urlPost = subSite.getUrlPost();
- this.incrementalPages = subSite.getIncrementalPages();
- this.incrementSleepTime = subSite.getIncrementSleepTime();
-
-
- return subSite;
- }
-
-
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java
deleted file mode 100644
index 9544a2eab..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java
+++ /dev/null
@@ -1,224 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.URL;
-import net.trustie.webmagic.utils.DBPipeline;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.LongSleepHelper;
-import net.trustie.webmagic.utils.Protector;
-import net.trustie.webmagic.utils.TableHelper;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.apache.log4j.xml.DOMConfigurator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.pipeline.Pipeline;
-import us.codecraft.webmagic.scheduler.QueueScheduler;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-import us.codecraft.webmagic.scheduler.component.BloomFilterDuplicateRemover;
-
-import javax.annotation.Resource;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Created by gyiang on 2014/11/15.
- */
-@Component
-public class DetailHtmlCrawler extends Thread{
- Logger logger = Logger.getLogger(this.getClass());
- private int batchSize = 10;
- @Resource
- private URLDao urlDao;
- @Resource
- private PointerDAO pointerDao;
-
- @Qualifier("detailHtmlPipeline")
- @Autowired
- private DBPipeline pipeline;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String urlTableName = "";
- private String urlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
-
- private String detailHtmlTableName = "";
- private String detailHtmlTableSql = "";
- private Configure conf = null;
-
- Spider spider;
- Site site;
- private void init(String configureName) {
- conf = new Configure(configureName);
- domain = conf.getDomain();
- detailHtmlTableName = domain + "_html_detail";
- urlTableName = domain + "_url";
- errorPageTableName = domain + "_error_page";
- initSql();
- initTable();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.urlTableSql = FileReader.readFile("./sql/url.sql");
- this.detailHtmlTableSql = FileReader.readFile("./sql/detail_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
- initOneTable(this.detailHtmlTableName, this.detailHtmlTableSql);
- initOneTable(this.urlTableName, this.urlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- public void crawl(String configureName) throws InterruptedException {
- // Configure conf = new Configure(configureName);
- this.init(configureName);
- // 每一批取线程数个url
- batchSize = conf.getThreadNum();
- Set acceptStatCode = new HashSet();
- //pipeline.setTableName(this.detailHtmlTableName);
- pipeline.setDetailHtmlTableName(this.detailHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
-
- // List proxies = Utils.getProxies(conf);
- site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- // .setHttpProxyPool(proxies);
- // int offset = 0;// conf.getOffset();
- // int limitLine = conf.getLimitLine();
-
-
- int startId = this.getPointer(this.urlTableName);
- List urls = new ArrayList();
- List urlList = new ArrayList();
-
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
-
- while (true) {
- //lzx
- /*spider = Spider
- .create(new DetailHtmlProcessor(site))
- .addPipeline(pipeline).setScheduler(new SimpleScheduler()).thread(conf.getThreadNum());*/
- // int endId = startId + this.batchSize;
- //urls = urlDao.getBatch(this.urlTableName, startId, this.batchSize);
- urls = urlDao.getBatch(this.urlTableName, startId, 1);
- //urlList = new ArrayList();
- System.out.println("urls: " + urls.size());
- for (URL url : urls) {
- urlList.add(url.getUrl());
- startId = url.getId();
- }
- if (urlList.size() == 0) {
- System.out.println("no url in queue--->Thread.sleep()");
- Thread.sleep(conf.getNoUrlWaitTime());
- continue;
- }
-
- //lzx
- /*spider.startUrls(urlList);
- spider.run();*/
- renewSpider();
- spider.startUrls(urlList);
- spider.run();
-
- pointerDao.updatePointer(this.pointersTableName, this.urlTableName,
- startId);
- urls.clear();
- urlList.clear();
-
- // sleep
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- }
-
- }
- }
-
- private int getPointer(String tableName) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName, 0);
- }
- return pointer;
-
- }
- private void renewSpider() {
-
- // .setHttpProxyPool(proxies);
- // .setCharset("UTF-8");
- Protector.setRandomArgs(site,conf);
- this.spider = Spider
- .create(new DetailHtmlProcessor(site))
- .addPipeline(this.pipeline)
-
- // .addUrl(conf.getStartUrl())
- // .setScheduler(new QueueScheduler().setDuplicateRemover(new
- // BloomFilterDuplicateRemover(10000))
- .thread(conf.getThreadNum())
- .setScheduler(new SimpleScheduler());
- }
- public static void main(String[] args) throws InterruptedException {
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final DetailHtmlCrawler htmlCrawler = applicationContext
- .getBean(DetailHtmlCrawler.class);
- htmlCrawler.crawl(configureName);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java
deleted file mode 100644
index 7ba947bca..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.utils.DBPipeline;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-
-import java.util.List;
-
-/**
- * Created by gyiang on 2014/11/16.
- */
-@Component("detailHtmlPipeline")
-public class DetailHtmlPipeline extends DBPipeline {
- Logger logger = Logger.getLogger(DetailHtmlPipeline.class);
- @Resource
- private DetailHtmlDao detailHtmlDao;
-
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- DetailHtml detailHtml = resultItems.get("model");
-
- try {
- if (detailHtml.getStatusCode() == 200) {
- detailHtmlDao.insertDetailHtml(detailHtml,
- this.getDetailHtmlTableName());
- logger.info("save page " + detailHtml.getUrl() + " success");
- } else {
- errorPageDao.insertPage(this.getErrorPageTableName(),
- detailHtml);
- logger.info("save error page " + detailHtml.getUrl() + "error"
- + detailHtml.getStatusCode() + " success!");
- }
- } catch (Exception e) {
- e.printStackTrace();
- // Logger logger = Logger.getLogger(MySqlPipelinePost.class);
- logger.error("error on Pipeline,when:" + detailHtml.getUrl());
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java
deleted file mode 100644
index d7b96bb2d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package net.trustie.webmagic.one;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.commons.codec.digest.DigestUtils;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-public class DetailHtmlProcessor implements PageProcessor {
- private Site site;
-
- // private String listHtmlTName;// listHtmlTableName
- // private String reList;
-
- public DetailHtmlProcessor(Site site) {
- this.site = site;
- }
-
- @Override
- public void process(Page page) {
-
- DetailHtml detailHtml = new DetailHtml();
- detailHtml.setStatusCode(page.getStatusCode());
- String html = page.getRawText();
- detailHtml.setHtml(html);
- String url = page.getRequest().getUrl();
- detailHtml.setUrl(url);
- if (url != null) {
- detailHtml.setUrlMd5(DigestUtils.md5Hex(page.getUrl().get()));
- }
- detailHtml.setType("Detail");
-
- detailHtml.setCrawledTime(Utils.getNow());
- if (html != null) {
- detailHtml.setPageMd5(DigestUtils.md5Hex(html));
- }
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", detailHtml);
- // page.putField("listTableName", listTableName);
- }
-
- @Override
- public Site getSite() {
- return site;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlSubProcessor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlSubProcessor.java
deleted file mode 100644
index 01dc7a4cc..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/DetailHtmlSubProcessor.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.log4j.Logger;
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-import us.codecraft.webmagic.selector.PlainText;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class DetailHtmlSubProcessor implements SubPageProcessor {
-
- private Logger logger = Logger.getLogger(DetailHtmlSubProcessor.class);
-
- // public static final String URL_POST =
- // "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";
-
- @Override
- public MatchOther processPage(Page page) {
-
-// SimpleDateFormat timestampFormat = new SimpleDateFormat(
-// "yyyyMMddHHmmss");
- SimpleDateFormat crawledTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- Date date = new Date();
- //String timestamp = timestampFormat.format(date);
- String crawledTime = crawledTimeFormat.format(date);
-
- DetailHtml detailHtml = new DetailHtml();
- if (page.getStatusCode() == 404 || page.getStatusCode() == 503) {
- detailHtml.setStatusCode(page.getStatusCode());
- //detailHtml.setTimestamp(page.getStatusCode() + "");
- page.setSkip(true);// 跳过这个页面
- } else {
- //detailHtml.setStatusCode(statusCode);
- }
-
- detailHtml.setHtml(page.getRawText().toString());
- detailHtml.setUrl(page.getUrl().get());
- detailHtml.setUrlMd5(DigestUtils.md5Hex(page.getUrl().get()));
- detailHtml.setType("Detail");
-
- /*
- * if(page.getUrl().regex(reList).match()) { htmlModel.setType("list");
- * }else if(page.getUrl().regex(rePost).match()) {
- * htmlModel.setType("post"); }else {
- * logger.error("unknown RE,current url:"+page.getUrl().get()); }
- */
-
- detailHtml.setCrawledTime(crawledTime);
- detailHtml.setPageMd5(DigestUtils.md5Hex(page.getRawText().toString()));
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", detailHtml);
- //page.putField("listTableName", listTableName);
-
- return MatchOther.YES;
- }
-
-// public DetailHtmlSubProcessor(String postTableName, String listTableName) {
-// this.postTableName = postTableName;
-// this.listTableName = listTableName;
-// }
-
- @Override
- public boolean match(Request page) {
- /*
- * if (new PlainText(page.getUrl()).regex(URL_POST).match()){ return
- * true; }else { return false; }
- */
- return true;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageCrawler.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageCrawler.java
deleted file mode 100644
index dbb2dd103..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageCrawler.java
+++ /dev/null
@@ -1,197 +0,0 @@
-package net.trustie.webmagic.one;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.annotation.Resource;
-
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.URL;
-import net.trustie.webmagic.utils.DBPipeline;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.TableHelper;
-
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-
-/**
- * Created by gyiang on 2014/11/15.
- */
-@Component
-public class ErrorPageCrawler {
- Logger logger = Logger.getLogger(this.getClass());
- private int batchSize = 10;
- @Resource
- private ErrorPageDao errorPageDao;
- @Resource
- private PointerDAO pointerDao;
-
- @Qualifier("errorPagePipeline")
- @Autowired
- private DBPipeline pipeline;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String urlTableName = "";
- private String urlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
-
- private String listHtmlTableName = "";
- private String listHtmlTableSql = "";
-
- private String detailHtmlTableName = "";
- private String detailHtmlTableSql = "";
- private Configure conf = null;
-
- private void init(String configureName) {
- conf = new Configure(configureName);
- domain = conf.getDomain();
- listHtmlTableName = domain + "_html_list";
- detailHtmlTableName = domain + "_html_detail";
- urlTableName = domain + "_url";
- errorPageTableName = domain + "_error_page";
- batchSize = conf.getThreadNum();
- pipeline.setListHtmlTableName(this.listHtmlTableName);
- pipeline.setDetailHtmlTableName(this.detailHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
- initSql();
- initTable();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.urlTableSql = FileReader.readFile("./sql/url.sql");
- this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
- this.detailHtmlTableSql = FileReader.readFile("./sql/detail_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
- initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
- initOneTable(this.detailHtmlTableName, this.detailHtmlTableSql);
- initOneTable(this.urlTableName, this.urlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- public void crawl(String configureName) throws InterruptedException {
- // Configure conf = new Configure(configureName);
- this.init(configureName);
- // 每一批取线程数个url
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
-
- // List proxies = Utils.getProxies(conf);
- Site site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- // .setHttpProxyPool(proxies);
- // int offset = 0;// conf.getOffset();
- // int limitLine = conf.getLimitLine();
- Spider spider = Spider.create(new ErrorPageProcessor(site))
- .addPipeline(pipeline).setScheduler(new SimpleScheduler())
- .thread(conf.getThreadNum());
-
- int startId = this.getPointer(this.errorPageTableName);
- List urls = new ArrayList();
- Request[] urlArray = null;
- Request request = null;
- Map maps = new HashMap();
-
- while (true) {
- // int endId = startId + this.batchSize;
- urls = errorPageDao.getBatch(this.errorPageTableName, startId,
- this.batchSize);
- urlArray = new Request[urls.size()];
- int i = 0;
- for (URL url : urls) {
- request = new Request(url.getUrl());
- maps.put("type", url.getType());
- request.setExtras(maps);
- urlArray[i] = request;
- startId = url.getId();
- i++;
- }
- i = 0;
- if (urlArray.length == 0) {
- System.out.println("no error url in queue--->Thread.sleep()");
- Thread.sleep(conf.getNoUrlWaitTime());
- continue;
- }
-
- spider.addRequest(urlArray);
- spider.run();
-
- pointerDao.updatePointer(this.pointersTableName,
- this.errorPageTableName, startId);
- urls.clear();
- // sleep
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- }
- }
-
- private int getPointer(String tableName) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName, 0);
- }
- return pointer;
-
- }
-
- public static void main(String[] args) throws InterruptedException {
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final ErrorPageCrawler errorPageCrawler = applicationContext
- .getBean(ErrorPageCrawler.class);
- errorPageCrawler.crawl(configureName);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPagePipeline.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPagePipeline.java
deleted file mode 100644
index 666ae6f82..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPagePipeline.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.AbstractHtml;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.DBPipeline;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-
-import java.util.List;
-
-/**
- * Created by gyiang on 2014/11/16.
- */
-@Component("errorPagePipeline")
-public class ErrorPagePipeline extends DBPipeline {
- Logger logger = Logger.getLogger(this.getClass());
- @Resource
- private DetailHtmlDao detailHtmlDao;
-
- @Resource
- private ListHtmlDao listHtmlDao;
-
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- AbstractHtml html = resultItems.get("model");
-
- try {
- if (html.getStatusCode() == 200) {
- if ("List".equals(html.getType())) {
- listHtmlDao.insertListHtml((ListHtml) html,
- this.getListHtmlTableName());
- } else if ("Detail".equals(html.getType())) {
- detailHtmlDao.insertDetailHtml((DetailHtml) html,
- this.getDetailHtmlTableName());
- }
- logger.info("save page " + html.getUrl() + " success!");
- } else {
- errorPageDao.insertPage(this.getErrorPageTableName(), html);
- logger.info("save error page " + html.getUrl() + "error"
- + html.getStatusCode() + " success!");
- }
- } catch (Exception e) {
- e.printStackTrace();
- // Logger logger = Logger.getLogger(MySqlPipelinePost.class);
- logger.error("error on Pipeline,when:" + html.getUrl());
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageProcessor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageProcessor.java
deleted file mode 100644
index e7528eb29..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ErrorPageProcessor.java
+++ /dev/null
@@ -1,85 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.log4j.Logger;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-/**
- * Created by gyiang on 2014/12/19.
- */
-public class ErrorPageProcessor implements PageProcessor {
- Logger logger = Logger.getLogger(this.getClass());
- private Site site;
-
- public ErrorPageProcessor(Site site) {
- this.site = site;
- }
-
- @Override
- public void process(Page page) {
-
- if ("List".equals((String) (page.getRequest().getExtra("type")))) {
- constructListHtml(page);
- } else if ("Detail"
- .equals((String) (page.getRequest().getExtra("type")))) {
- constructDetailHtml(page);
- } else {
- logger.info("errorPage type error :" + page.getRequest().getUrl());
- }
- }
-
- private void constructListHtml(Page page) {
- ListHtml listHtml = new ListHtml();
- listHtml.setStatusCode(page.getStatusCode());
- String html = "";
- html = page.getRawText();
- listHtml.setHtml(html);
-
- listHtml.setCrawledTime(Utils.getNow());
- String url = page.getUrl().get();
- listHtml.setUrl(url);
- listHtml.setUrlMd5(DigestUtils.md5Hex(url));
- listHtml.setPageMd5(DigestUtils.md5Hex(html));// 用于确定是已经抽过
- listHtml.setType("List");
- page.putField("model", listHtml);
- }
-
- private void constructDetailHtml(Page page) {
- DetailHtml detailHtml = new DetailHtml();
-
- detailHtml.setStatusCode(page.getStatusCode());
-
- detailHtml.setHtml(page.getRawText().toString());
- detailHtml.setUrl(page.getUrl().get());
- detailHtml.setUrlMd5(DigestUtils.md5Hex(page.getUrl().get()));
- detailHtml.setType("Detail");
-
- /*
- * if(page.getUrl().regex(reList).match()) { htmlModel.setType("list");
- * }else if(page.getUrl().regex(rePost).match()) {
- * htmlModel.setType("post"); }else {
- * logger.error("unknown RE,current url:"+page.getUrl().get()); }
- */
-
- detailHtml.setCrawledTime(Utils.getNow());
- detailHtml.setPageMd5(DigestUtils.md5Hex(page.getRawText().toString()));
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", detailHtml);
- }
-
- @Override
- public Site getSite() {
- return site;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlProcessor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlProcessor.java
deleted file mode 100644
index ec3ddcf94..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlProcessor.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.HtmlSubProcessorList;
-import net.trustie.webmagic.one.DetailHtmlSubProcessor;
-import net.trustie.webmagic.utils.LoadConf;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class HtmlProcessor extends CompositePageProcessor {
-
- private Site site = null;
- private String regexUrl;
- private String sourceRegion;
-
- public HtmlProcessor(Site site, String regexUrlList, String regexUrlPost,
- String listTableName) {
-
- super(site);
- this.site = site;
- // super.addSubPageProcessor(new HtmlSubProcessorPost());
- super.addSubPageProcessor(new HtmlSubProcessorList(regexUrlList,
- regexUrlPost, listTableName));
- }
-
- /*
- * @Override public void process(Page page) {
- * page.addTargetRequests(page.getHtml
- * ().xpath(sourceRegion).links().regex(regexUrl).all());
- *
- * }
- */
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlSubProcessorList.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlSubProcessorList.java
deleted file mode 100644
index 0911bf8f5..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/HtmlSubProcessorList.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package net.trustie.webmagic.one;
-
-import org.apache.log4j.Logger;
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-
-import java.util.List;
-
-/**
- * Created by gan on 2014/11/15. Notified by gan on 2014/11/
- */
-public class HtmlSubProcessorList implements SubPageProcessor {
- /*
- * 最初使用static final方法 public static final String URL_LIST; public static
- * final String URL_POST ;
- */
- private String urlList;
- private String urlPost;
- private String listTableName;
-
- public HtmlSubProcessorList(String urlList, String urlPost,
- String listTableName) {
- this.urlList = urlList;
- this.urlPost = urlPost;
- this.listTableName = listTableName;
- }
-
- @Override
- public MatchOther processPage(Page page) {
-
- // 加入爬取队列,策略改变后,变成存入数据库
- /*
- * page.addTargetRequests(page.getHtml().xpath(
- * "//div[@class=\"articleList\"]").links().regex(urlPost).all());
- * page.addTargetRequests(page.getHtml().links().regex(urlList).all());
- * page.getTargetRequests()
- */
- // 列表页链接加入,发现新的列表页
-
- // System.out.println(page.getRawText());
- page.addTargetRequests(page.getHtml().links().regex(urlList).all());
- // 获取post链接
- List targetUrl = page.getHtml().links().regex(urlPost).all();
- // 列表页的url也存入,去爬
- targetUrl.add(page.getUrl().get());
-
- // 交给pipeline
- page.putField("findUrl", targetUrl);
- page.putField("listTableName", listTableName);
-
- // 只取列表页,跳过持久化过程
- // page.setSkip(false);
- return MatchOther.YES;
- }
-
- @Override
- public boolean match(Request page) {
- return PatternUrl.match(page, urlList);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java
deleted file mode 100644
index f04be9297..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java
+++ /dev/null
@@ -1,508 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.CreateTableDAO;
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.utils.DBPipeline;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.LongSleepHelper;
-import net.trustie.webmagic.utils.MyRandomer;
-import net.trustie.webmagic.utils.OrderSpider;
-import net.trustie.webmagic.utils.Protector;
-import net.trustie.webmagic.utils.SubSite;
-import net.trustie.webmagic.utils.TableHelper;
-import net.trustie.webmagic.utils.UserAgentPool;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.http.HttpHost;
-import org.apache.ibatis.binding.BindingException;
-import org.apache.log4j.Logger;
-import org.apache.log4j.xml.DOMConfigurator;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.monitor.SpiderMonitor;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.pipeline.Pipeline;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-
-import javax.annotation.Resource;
-import javax.management.JMException;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-import java.io.File;
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-
-
-/**
- * Created by Administrator on 2014/12/19.
- */
-@Component
-public class ListHtmlCrawler {
-
-
-
- Logger logger = Logger.getLogger(this.getClass());
- @Qualifier("listHtmlPipeline")
- @Autowired
- private DBPipeline pipeline;
- @Resource
- private PointerDAO pointerDao;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
-// private String listHtmlTableName = "";
- private String listHtmlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
- private String pageIndexName = "";
- private String modeColName = "";
- // 记录列表页爬取位置的字段名
-
- private int startIndex = 0;
- private int endIndex = 0;
- private int mode;
-
- private Configure conf = null;
- private Set acceptStatCode = null;
- private Site site = null;
- private Spider spider = null;
- private Random random = new Random(66);
- private void init(String configureName) {
- this.conf = new Configure(configureName);
- this.acceptStatCode = this.initAcceptStatCode();
- this.domain = conf.getDomain();
-// this.listHtmlTableName = this.domain + "_html_list";
- this.errorPageTableName = this.domain + "_error_page";
- // 轮换注释掉 this.pageIndexName = this.domain + "_page_index";
- this.modeColName = this.domain + "_mode";
-
-
- initSql();
- // 初始化表
- initTable();
-
- // 轮换注释掉 startIndex = this.getPointer(pageIndexName,this.conf.getStartPageIndex());
- mode = this.getMode(this.modeColName, this.conf.getMode());
-
-// pipeline.setListHtmlTableName(this.listHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
-
- // List proxies = Utils.getProxies(conf);
- this.site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode)
- .addHeader("confname", configureName);
- //TO BE SEEN
- renewSpider();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
-// this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
-// initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
-
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- // 取得指针
- private int getPointer(String tableName, int defaultValue) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- return pointer + 1;
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName,
- defaultValue - 1);
- }
- return defaultValue;
-
- }
-
- private int getMode(String tableName, int defaultValue) {
- int pointer = 0;
- try {
- pointer = pointerDao.readPointer(this.pointersTableName, tableName);
- return pointer;
- } catch (BindingException e) {
- pointerDao.insertPointer(this.pointersTableName, tableName,
- defaultValue);
- }
- return defaultValue;
- }
-
- public void crawl(String configureName) {
- this.init(configureName);
-
- this.acceptStatCode = this.initAcceptStatCode();
-// pipeline.setTableName(this.listHtmlTableName);
- pipeline.setErrorPageTableName(this.errorPageTableName);
-
- this.startSpider( );
- }
-
- private List getOrderedUrl(String purl, String surl, int per,
- int incIndex, int startIndex) {
- List urls = new ArrayList();
- for (int g = startIndex; g < startIndex + per; g++) {
- int pageindex = g + incIndex;
- String url = purl + pageindex + surl;
- urls.add(url);
- // System.out.println(url);
- }
- return urls;
- }
-
- private void startSpider() {
- // List urls = new ArrayList();
- if (this.mode == 0) { // 镜像模式
- this.mirrorModeCrawl();
- logger.info("change to increment mode!!!");
- pointerDao.updatePointer(this.pointersTableName, this.modeColName,
- 1);
-
- this.reInit();
- this.incrementModeCrawl();
- } else if (this.mode == 1) { // 增量模式
- this.incrementModeCrawl();
- } else {
- logger.info("mode error!!!");
- }
- }
-
- private void reInit() {
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, this.conf.getStartPageIndex() - 1);//增量结束归起始页 ;
- //this.startIndex = this.conf.getStartPageIndex();
- }
-
-
-
-
- // 镜像模式爬取
- private void mirrorModeCrawl() {
-
-
-
- //star modify for 轮换
- int subSiteNum = this.conf.getSubSiteNum();
-
- logger.info("该站点共有" + subSiteNum + "个子站点");
- int doneNum = 0;//记录已完成镜像爬取的子站点个数
- int processNum = 10;//对每个子站点一次处理多少个页面
- while(doneNum < subSiteNum){
-
- //conf 负责获得这次处理的站点
- SubSite subSite = this.conf.nexSite();
- if(subSite.getName().equals(""))
- this.pageIndexName = this.domain +"_page_index";
- else
- this.pageIndexName = this.domain + "_" + subSite.getName()+ "_page_index";
-
- startIndex = this.getPointer(pageIndexName,this.conf.getStartPageIndex());
-
- List urls = new ArrayList();
- this.endIndex = this.conf.getEndInPageIndex();
-
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
- int j =0;
- int i = 0;
- for ( i = this.startIndex;( i <= this.endIndex ) && (j < processNum); i += conf.getPageF(),j++) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
-
- //对爬虫的参数从新进行随机化设置
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- }
- } catch (InterruptedException e) {
-
- e.printStackTrace();
- }
- }
-
- if(i > this.endIndex)
- doneNum++;//此站点已经处理完毕
-
-
- }
-
-
-
- //end of star modify 轮换
- /*List urls = new ArrayList();
- this.endIndex = this.conf.getEndInPageIndex();
-
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
-
- for (int i = this.startIndex; i <= this.endIndex; i += conf.getPageF()) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
-
- //对爬虫的参数从新进行随机化设置
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000 + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- }
- } catch (InterruptedException e) {
-
- e.printStackTrace();
- }
- }*/
- }
-
- private void renewSpider() {
-
- // .setHttpProxyPool(proxies);
- // .setCharset("UTF-8");
- Protector.setRandomArgs(site,conf);
-
- //processor和pipeline的入口
- this.spider = Spider
- .create(new ListHtmlProcessor(site))
- // .addPipeline(this.pipeline)
-
- // .addUrl(conf.getStartUrl())
- // .setScheduler(new QueueScheduler().setDuplicateRemover(new
- // BloomFilterDuplicateRemover(10000))
- .thread(conf.getThreadNum())
- .setScheduler(new SimpleScheduler());
- }
-
- // 增量模式爬取
- private void incrementModeCrawl() {
-
- List urls = new ArrayList();
-
-
-
- while (true) {
-
- //star modify for 轮换
- int subSiteNum = this.conf.getSubSiteNum();
- logger.info("该站点共有" + subSiteNum + "个子站点");
- int doneNum = 0;//记录已完成镜像爬取的子站点个数
- int processNum = 10;//对每个子站点一次处理多少个页面
- while(doneNum < subSiteNum){
- //conf 负责获得这次处理的站点
- SubSite subSite = this.conf.nexSite();
-
- if(subSite.getName().equals(""))
- this.pageIndexName = this.domain +"_page_index";
- else
- this.pageIndexName = this.domain + "_" + subSite.getName()+ "_page_index";
-
- startIndex = this.getPointer(pageIndexName,this.conf.getStartPageIndex());
-
- this.endIndex = this.conf.getIncrementalPages();
- //System.out.println("增量爬取" + startIndex + "~" + endIndex );
- if(startIndex >= endIndex)
- startIndex = this.conf.getStartPageIndex();//防止异常退出后,转换为增量或增量结束后指针没有来得及更新为起始页
- //System.out.println("增量爬取" + startIndex + "~" + endIndex );
- LongSleepHelper lsh = new LongSleepHelper(this.conf);
- lsh.startMonitor();
- int j =0;
- int i = 0;
- for (i = this.startIndex;( i <= this.endIndex ) && (j < processNum); i += conf.getPageF(),j++) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
-
- if(lsh.timeToLongSleep()){
- int longSleepTime = lsh.longsleepTime();
- logger.info("长时间段的随机休息 " + longSleepTime / 1000
- + "s");
- Thread.sleep(longSleepTime);
- }else{
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000
- + "s");
-
- Thread.sleep(conf.getSleepTimeSpider());
- }
-
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- if(i > this.endIndex){
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, this.conf.getStartPageIndex() - 1);//增量结束归起始页
- doneNum++;//此站点已经处理完毕
- }
-
- }
-
- this.reInit();
- logger.info("increment finish!!!");
- logger.info("sleep "+conf.getIncrementSleepTime()+"s");
- try {
- Thread.sleep(conf.getIncrementSleepTime()*1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- //this.reInit();
- }
-
-
- }
-
- private void incrementModeCrawlHelper(){
- List urls = new ArrayList();
- this.endIndex = this.conf.getIncrementalPages();
- // System.out.println(startIndex);
- // System.out.println(endIndex);
-
-
- //压力分散做法行不通,因为列表页的内容是不断的在变化的,此时的第一页和一个小时前的第一页已经大不一样了
- //如果一天爬取完,那平均多少秒爬一个
- //int baseTime = (24 * 60 * 60 * 1000) / this.endIndex;
- //设置爬虫新的随机区间,请求间隔不要太小,设置为一半,考虑到其他的消耗,也不要太长,设置为2/3
- //this.conf.setMinInterval(baseTime / 2);
- //this.conf.setMaxInterval(baseTime*5 / 6);
-
-
- // while (true) {
- for (int i = this.startIndex; i <= this.endIndex; i += conf.getPageF()) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- // 获得链接
- urls = getOrderedUrl(conf.getPrefix(), conf.getPostfix(),
- conf.getPageF(), 0, i);
- renewSpider();
-
- spider.startUrls(urls);
- spider.run();
- // 更新指针
- pointerDao.updatePointer(this.pointersTableName,
- this.pageIndexName, i + conf.getPageF() - 1);
- // sleep
- try {
- logger.info("sleep " + conf.getSleepTimeSpider() / 1000
- + "s");
- Thread.sleep(conf.getSleepTimeSpider());
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- logger.info("increment finish!!!");
- // logger.info("sleep "+conf.getIncrementSleepTime()+"s");
- /*try {
- Thread.sleep(conf.getIncrementSleepTime()*1000);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- */
- this.reInit();
- // }
- }
- private Set initAcceptStatCode() {
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
- return acceptStatCode;
- }
-
- public static void main(String[] args) throws InterruptedException {
- // DOMConfigurator.configure("log4j.xml");
- // System.err.println(new File("log4j.xml").getAbsolutePath());
- String configureName = "";
- if (args.length != 0)
- configureName = args[0].toString();
- if (configureName.equals("")) {
- configureName = "ossean";
- } else {
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
- final ListHtmlCrawler htmlCrawler = applicationContext
- .getBean(ListHtmlCrawler.class);
- htmlCrawler.crawl(configureName);
- System.out.println("this is in th crawler");
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlPipeline.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlPipeline.java
deleted file mode 100644
index 4a988ec60..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlPipeline.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.DBPipeline;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-
-/**
- * Created by ganyiang on 2014/12/19.
- */
-@Component("listHtmlPipeline")
-public class ListHtmlPipeline extends DBPipeline {
- Logger logger = Logger.getLogger(ListHtmlPipeline.class);
- @Resource
- private ListHtmlDao listHtmlDao;
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- ListHtml listHtml = resultItems.get("model");
-
- try {
- if (listHtml.getStatusCode() == 200) {
- listHtmlDao.insertListHtml(listHtml,
- this.getListHtmlTableName());
- logger.info("save page " + listHtml.getUrl());
- } else {
- errorPageDao.insertPage(this.getErrorPageTableName(), listHtml);
- logger.info("save error page " + listHtml.getUrl() + " error "
- + listHtml.getStatusCode()+" success!");
- }
- } catch (Exception e) {
- e.printStackTrace();
- // Logger logger = Logger.getLogger(MySqlPipelinePost.class);
- logger.error("error on Pipeline,when:" + listHtml.getUrl());
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java
deleted file mode 100644
index 13b11601f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.extrator.CSSPathExtractor;
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.URLPatternExtractor;
-import net.trustie.webmagic.one.dao.ErrorPageDao;
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.dao.PointerDAO;
-import net.trustie.webmagic.one.dao.URLDao;
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-import net.trustie.webmagic.utils.FileReader;
-import net.trustie.webmagic.utils.TableHelper;
-import net.trustie.webmagic.utils.Utils;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.log4j.Logger;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.processor.PageProcessor;
-import us.codecraft.webmagic.selector.Html;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.Set;
-
-import javax.annotation.Resource;
-
-
-
-/**
- * Created by gyiang on 2014/12/19.
- */
-public class ListHtmlProcessor implements PageProcessor {
-
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "classpath:/spring/applicationContext*.xml");
-
- private Site site;
-// private String conf;
-
- Extractor extractor = null;
-
- Logger logger = Logger.getLogger(this.getClass());
- private static int batchSize = 10;
-// @Resource
-// private ListHtmlDao listHtmlDao;
- @Resource
- private PointerDAO pointerDao;
- @Resource
- private URLDao urlDao;
- @Resource
- private ErrorPageDao errorPageDao;
-
- @Qualifier("tableHelper")
- @Autowired
- private TableHelper tableHelper;
-
- private String pointersTableName = "pointers";
- private String pointersSql = "";
-
- private String domain = "";
- private String urlTableName = "";
- private String urlTableSql = "";
- private String errorPageTableName = "";
- private String errorPageTableSql = "";
-
- private String listHtmlTableName = "";
- private String listHtmlTableSql = "";
- private Configure conf = null;
-
- private void init(String configureName) {
- conf = new Configure(configureName);
- domain = conf.getDomain();
- listHtmlTableName = domain + "_html_list";
- urlTableName = domain + "_url";
- errorPageTableName = domain + "_error_page";
- initSql();
- initTable();
- }
-
- private void initSql() {
- this.pointersSql = FileReader.readFile("./sql/pointers.sql");
- this.urlTableSql = FileReader.readFile("./sql/url.sql");
-// this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
- this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
- }
-
- private void initTable() {
- initOneTable(this.pointersTableName, this.pointersSql);
-// initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
- initOneTable(this.urlTableName, this.urlTableSql);
- initOneTable(this.errorPageTableName, this.errorPageTableSql);
- }
-
- private void initOneTable(String table, String fields) {
- tableHelper=applicationContext.getBean(TableHelper.class);
- if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-
- tableHelper.createTable(table, fields);
- logger.info("create table " + table + " success!!!");
- }
- }
-
- public ListHtmlProcessor(Site site) {
- this.site = site;
- }
-
-
- @Override
- public void process(Page page) {
-
- urlGet(page);
-
-// String confname=site.getHeaders().get("confname");
-// //System.out.println(confname);
-// ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
-// "classpath:/spring/applicationContext*.xml");
-// final UrlExtractor2 urlExtractor = applicationContext
-// .getBean(UrlExtractor2.class);
-// try {
-// urlExtractor.exec(confname);
-// } catch (InterruptedException e) {
-// // TODO Auto-generated catch block
-// e.printStackTrace();
-// }
-
-
- /*
- ListHtml listHtml = new ListHtml();
-
- listHtml.setStatusCode(page.getStatusCode());
- String html = page.getRawText();
- listHtml.setHtml(html);
-
- listHtml.setCrawledTime(Utils.getNow());
- String url;// = page.getUrl().get();
- url = page.getRequest().getUrl();
- listHtml.setUrl(url);
- if (url != null) {
- listHtml.setUrlMd5(DigestUtils.md5Hex(url));
- }
- if (html != null) {
- listHtml.setPageMd5(DigestUtils.md5Hex(html));// 用于确定是已经抽过
- }
- listHtml.setType("List");
-
- // 存入扩展字段,后面持久化Pipeline调用
- page.putField("model", listHtml);
- */
-
- }
-
- @Override
- public Site getSite() {
- return site;
- }
-
- public void urlGet(Page page){
-
- String confname=site.getHeaders().get("confname");
-
- this.init(confname);
- conf = new Configure(confname);
-
- Set urlSet = new HashSet();
- String error = null;
- String extractedTime = "";
-
- Extractor extractor = null;
-
- if ("URLPattern".equals(conf.getExtractMethod())) {
- extractor = new URLPatternExtractor(conf.getUrlPost());
- } else if ("CSSPath".equals(conf.getExtractMethod())) {
- extractor = new CSSPathExtractor(conf.getUrlPost());
- } else {
- logger.error("extract method error!!!");
- }
-
- SimpleDateFormat extractedTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- Date date = new Date();
- extractedTime = extractedTimeFormat.format(date);
-
- Html html = page.getHtml();
- urlSet = extractor.extract(html);
-
-// String pageMd5 = DigestUtils.md5Hex(html);
- urlDao=applicationContext.getBean(URLDao.class);
- try {
- for (String url : urlSet) {
- try {
- urlDao.addURL(urlTableName, url, extractedTime);
- } catch (Exception e) {
- logger.error(e.toString());
- error = "500";
- }
- }
- } catch (Exception e) {
- logger.error(e.getMessage());
- }
-
- urlSet.clear();
-
-
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MianThread.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MianThread.java
deleted file mode 100644
index a603de265..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MianThread.java
+++ /dev/null
@@ -1,18 +0,0 @@
-//package net.trustie.webmagic.one;
-//
-//public class MianThread {
-//
-// public static void main(String[] args) throws InterruptedException {
-// UrlExtractor url=new UrlExtractor(args[0]);
-// url.start();
-//
-//
-//
-// Thread.sleep(100000);
-// url.destroy();
-//
-//
-//
-// }
-//
-//}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MySqlPipelineList.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MySqlPipelineList.java
deleted file mode 100644
index e6be3c981..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/MySqlPipelineList.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package net.trustie.webmagic.one;
-
-import net.trustie.webmagic.one.dao.DetailHtmlDao;
-import net.trustie.webmagic.one.dao.URLDao;
-import org.springframework.stereotype.Component;
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-import javax.annotation.Resource;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-
-/**
- * Created by gyiang on 2014/11/16.
- */
-@Component("MySqlPipelineList")
-public class MySqlPipelineList implements Pipeline {
- @Resource
- private URLDao htmlTargetUrlDao;
-
- @Override
- public void process(ResultItems resultItems, Task task) {
- List l = (List) resultItems.get("findUrl");
- // 爬取时间戳
- SimpleDateFormat bartDateFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- String crawlerTime = bartDateFormat.format(new Date());
- for (String urlPost : l) {
- htmlTargetUrlDao.add(urlPost, crawlerTime,
- resultItems.get("listTableName").toString());
- }
-
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/PatternUrl.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/PatternUrl.java
deleted file mode 100644
index 1b7fb89f3..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/PatternUrl.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package net.trustie.webmagic.one;
-
-import us.codecraft.webmagic.Request;
-
-import java.util.regex.Pattern;
-
-/**
- * Created by gyiang on 2014/11/19.
- */
-public class PatternUrl {
- protected String pattern;
- private Pattern patternCompiled;
-
- public static boolean match(Request request, String pattern) {
- return Pattern.compile(pattern).matcher(request.getUrl()).matches();
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/UrlExtractor.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/UrlExtractor.java
deleted file mode 100644
index e2d4001c7..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/UrlExtractor.java
+++ /dev/null
@@ -1,259 +0,0 @@
-//package net.trustie.webmagic.one;
-//
-//import net.trustie.webmagic.extrator.CSSPathExtractor;
-//import net.trustie.webmagic.extrator.Extractor;
-//import net.trustie.webmagic.extrator.URLPatternExtractor;
-//import net.trustie.webmagic.one.dao.ErrorPageDao;
-//import net.trustie.webmagic.one.dao.URLDao;
-//import net.trustie.webmagic.one.dao.PointerDAO;
-//import net.trustie.webmagic.one.dao.DetailHtmlDao;
-//import net.trustie.webmagic.one.dao.ListHtmlDao;
-//import net.trustie.webmagic.one.model.ListHtml;
-//import net.trustie.webmagic.utils.FileReader;
-//import net.trustie.webmagic.utils.TableHelper;
-//
-//import org.apache.commons.codec.digest.DigestUtils;
-//import org.apache.ibatis.binding.BindingException;
-//import org.apache.log4j.Logger;
-//import org.apache.log4j.xml.DOMConfigurator;
-//import org.springframework.beans.factory.annotation.Autowired;
-//import org.springframework.beans.factory.annotation.Qualifier;
-//import org.springframework.context.ApplicationContext;
-//import org.springframework.context.support.ClassPathXmlApplicationContext;
-//import org.springframework.stereotype.Component;
-//
-//import us.codecraft.webmagic.Page;
-//import us.codecraft.webmagic.model.annotation.ExtractByUrl;
-//import us.codecraft.webmagic.selector.Html;
-//import us.codecraft.webmagic.utils.UrlUtils;
-//
-//import javax.annotation.Resource;
-//
-//import java.io.File;
-//import java.text.SimpleDateFormat;
-//import java.util.ArrayList;
-//import java.util.Date;
-//import java.util.HashSet;
-//import java.util.List;
-//import java.util.Set;
-//
-///**
-// * Created by Administrator on 2014/12/21.
-// */
-//@Component
-//public class UrlExtractor extends Thread {
-// Logger logger = Logger.getLogger(this.getClass());
-// private static int batchSize = 10;
-// @Resource
-// private ListHtmlDao listHtmlDao;
-// @Resource
-// private PointerDAO pointerDao;
-// @Resource
-// private URLDao urlDao;
-// @Resource
-// private ErrorPageDao errorPageDao;
-//
-// @Qualifier("tableHelper")
-// @Autowired
-// private TableHelper tableHelper;
-//
-// private String pointersTableName = "pointers";
-// private String pointersSql = "";
-//
-// private String domain = "";
-// private String urlTableName = "";
-// private String urlTableSql = "";
-// private String errorPageTableName = "";
-// private String errorPageTableSql = "";
-//
-// private String listHtmlTableName = "";
-// private String listHtmlTableSql = "";
-// private Configure conf = null;
-//
-// private void init(String configureName) {
-// conf = new Configure(configureName);
-// domain = conf.getDomain();
-// listHtmlTableName = domain + "_html_list";
-// urlTableName = domain + "_url";
-// errorPageTableName = domain + "_error_page";
-// initSql();
-// initTable();
-// }
-//
-// private void initSql() {
-// this.pointersSql = FileReader.readFile("./sql/pointers.sql");
-// this.urlTableSql = FileReader.readFile("./sql/url.sql");
-// this.listHtmlTableSql = FileReader.readFile("./sql/list_html.sql");
-// this.errorPageTableSql = FileReader.readFile("./sql/error_page.sql");
-// }
-//
-// private void initTable() {
-// initOneTable(this.pointersTableName, this.pointersSql);
-// initOneTable(this.listHtmlTableName, this.listHtmlTableSql);
-// initOneTable(this.urlTableName, this.urlTableSql);
-// initOneTable(this.errorPageTableName, this.errorPageTableSql);
-// }
-//
-// private void initOneTable(String table, String fields) {
-//
-// if (!tableHelper.isTableExist(table)) { // 表不存在即初始化
-//
-// tableHelper.createTable(table, fields);
-// logger.info("create table " + table + " success!!!");
-// }
-// }
-//
-// public void exec(String configureName) throws InterruptedException {
-// this.init(configureName);
-// int startId = this.getPointer(listHtmlTableName);
-// List list = new ArrayList();
-// Set urlSet = new HashSet();
-//
-// Extractor extractor = null;
-// if ("URLPattern".equals(conf.getExtractMethod())) {
-// extractor = new URLPatternExtractor(conf.getUrlPost());
-// } else if ("CSSPath".equals(conf.getExtractMethod())) {
-// extractor = new CSSPathExtractor(conf.getUrlPost());
-// } else {
-// logger.error("extract method error!!!");
-// }
-// SimpleDateFormat extractedTimeFormat = new SimpleDateFormat(
-// "yyyy-MM-dd HH:mm:ss");
-// Date date = null;
-// String extractedTime = "";
-// Html html = null;
-// String fixRelativeHref = conf.getFixAllRelativeHrefs();
-// while(true) {
-// //Thread.sleep(100);
-// // int startId = start+UrlExtractor.batchSize;
-// // int endId = startId + UrlExtractor.batchSize;
-// list = listHtmlDao.getBatch(listHtmlTableName, startId,
-// UrlExtractor.batchSize);
-// if (list.size() == 0) {
-// System.out.println("no listHtml in queue--->Thread.sleep()");
-// Thread.sleep(conf.getNoUrlWaitTime());
-// continue;
-// }
-// for (ListHtml listHtml : list) {
-// if (listHtml == null || listHtml.getHtml() == "") {
-// continue;// 跳过
-// }
-// // SimpleDateFormat timestampFormat = new SimpleDateFormat(
-// // "yyyyMMddHHmmss");
-// date = new Date();
-// // String timestamp = timestampFormat.format(date);
-// extractedTime = extractedTimeFormat.format(date);
-//
-// html = new Html(UrlUtils.fixAllRelativeHrefs(
-// listHtml.getHtml(), fixRelativeHref));
-//
-// urlSet = extractor.extract(html);
-//
-// String pageMd5 = DigestUtils.md5Hex(listHtml.getHtml());// html.get()
-// // same
-// // rawText
-// String error = null;
-//
-// try {
-// for (String url : urlSet) {
-// try {
-// urlDao.addURL(urlTableName, url, extractedTime);
-// } catch (Exception e) {
-// logger.error(e.getMessage());
-// error = "500";
-// }
-// }
-// // 存完一次后标记
-// if (error != null) {
-// // pointerDao.updatePointer(conf.getListHtmlTableName(),
-// // listHtml.getId());
-// // listHtmlDao.timestamp(listHtmlTableName, pageMd5,
-// // timestamp);
-// } else {// 成功
-// pointerDao.updatePointer(this.pointersTableName,
-// listHtmlTableName, listHtml.getId());
-// // listHtmlDao
-// // .timestamp(listHtmlTableName, pageMd5, error);
-// }
-// } catch (Exception e) {
-// logger.error(e.getMessage());
-//
-// }
-//
-// logger.info("extracted id = " + listHtml.getId() + " "
-// + listHtml.getUrl());
-//
-// startId = listHtml.getId();
-// urlSet.clear();
-//
-// }
-// list.clear();
-// // startId=
-// }
-// }
-//
-// private int getPointer(String tableName) {
-// int pointer = 0;
-// try {
-// pointer = pointerDao.readPointer(this.pointersTableName, tableName);
-// } catch (BindingException e) {
-// pointerDao.insertPointer(this.pointersTableName, tableName, 0);
-// }
-// return pointer;
-//
-// }
-//
-//// public static void main(String[] args) throws InterruptedException {
-//// // DOMConfigurator.configure("log4j.xml");
-//// // System.err.println(new File("log4j.xml").getAbsolutePath());
-//// String configureName = "";
-//// if (args.length != 0)
-//// configureName = args[0].toString();
-//// if (configureName.equals("")) {
-//// configureName = "ossean";
-//// } else {
-//// }
-//// ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
-//// "classpath:/spring/applicationContext*.xml");
-//// final UrlExtractor urlExtractor = applicationContext
-//// .getBean(UrlExtractor.class);
-//// urlExtractor.exec(configureName);
-//// // urlExtrator.crawl(configureName);
-//// }
-//
-//
-// private String args;
-//
-// public UrlExtractor( String args) {
-// this.args=args;
-// }
-//
-//
-// public UrlExtractor( ) {
-// this.args=args;
-// }
-//
-//
-//
-//
-// @Override
-// public void run() {
-// String configureName =args;
-//
-// ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
-// "classpath:/spring/applicationContext*.xml");
-// final UrlExtractor urlExtractor = applicationContext
-// .getBean(UrlExtractor.class);
-// try {
-// urlExtractor.exec(configureName);
-// } catch (InterruptedException e) {
-// // TODO Auto-generated catch block
-// e.printStackTrace();
-// }
-//
-//
-// }
-//
-//
-//
-//}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/CreateTableDAO.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/CreateTableDAO.java
deleted file mode 100644
index d3208144d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/CreateTableDAO.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import java.util.List;
-
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-public interface CreateTableDAO {
- //判断表是否存在
- @Select("select 1 from ${tableName}")
- public List isTableExist(@Param("tableName") String tableName);
-
- @Update("create table if not exists ${table} ${fields}")
- public int createTable(@Param("table") String table, @Param("fields") String fields);
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java
deleted file mode 100644
index 723622ed3..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import java.util.List;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-
-import org.apache.ibatis.annotations.Delete;
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-
-/**
- * Created by Administrator on 2014/11/16.
- */
-public interface DetailHtmlDao {
-
- @Insert("insert into "
- + "${tableName}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{html},#{url},#{crawledTime},#{history},#{pageMd5},#{urlMd5})")
- public int add(DetailHtml detailHtml);
-
- @Insert("insert into "
- + "${table2Name}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{html},#{url},#{crawledTime},#{history},#{pageMd5},#{urlMd5})")
- public int addListHtml(DetailHtml detailHtml);
-
- @Delete("delete from ${tableName} where url=#{url}")
- public int delete404(@Param("url") String url,
- @Param("tableName") String tableName);
-
- @Select("select * from ${table} where id > #{startId} AND id <= #{endId}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("endId") int endId);
-
- @Insert("insert into "
- + "${table}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{detailHtml.html},#{detailHtml.url},#{detailHtml.crawledTime},#{detailHtml.history},#{detailHtml.pageMd5},#{detailHtml.urlMd5})")
- public int insertDetailHtml(@Param("detailHtml")DetailHtml detailHtml, @Param("table") String table);
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorPageDao.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorPageDao.java
deleted file mode 100644
index d3c2ae990..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorPageDao.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import java.util.List;
-
-import net.trustie.webmagic.one.model.AbstractHtml;
-import net.trustie.webmagic.one.model.URL;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-
-public interface ErrorPageDao {
-
- @Insert("insert into "
- + "${table}"
- + " (`html`,`url`,`crawledTime`,`pageMd5`,`urlMd5`,`type`) values (#{html.html},#{html.url},#{html.crawledTime},#{html.pageMd5},#{html.urlMd5},#{html.type})")
- public int insertPage(@Param("table") String table, @Param("html")AbstractHtml html);
-
- @Select("select * from ${table} where id > #{startId} LIMIT #{batchSize}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("batchSize") int batchSize);
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorURLDAO.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorURLDAO.java
deleted file mode 100644
index 0c989c419..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ErrorURLDAO.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import net.trustie.webmagic.one.model.URL;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-
-public interface ErrorURLDAO {
- @Insert("insert ${table} (`id`,`url`) values (#{id},#{url})")
- public int insertURL(@Param("table") String table, URL url);
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ListHtmlDao.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ListHtmlDao.java
deleted file mode 100644
index 552da0db6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/ListHtmlDao.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import net.trustie.webmagic.one.model.DetailHtml;
-import net.trustie.webmagic.one.model.ListHtml;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-import java.util.List;
-
-/**
- * Created by ganyiang on 2014/12/19.
- */
-public interface ListHtmlDao {
- @Insert("insert into "
- + "${table}"
- + " (`html`,`url`,`crawledTime`,`history`,`pageMd5`,`urlMd5`) values (#{listHtml.html},#{listHtml.url},#{listHtml.crawledTime},#{listHtml.history},#{listHtml.pageMd5},#{listHtml.urlMd5})")
- public int insertListHtml(@Param("listHtml")ListHtml listHtml, @Param("table") String table);
-
- @Select("select html from ${listTableName} where extracted is null limit #{startLine},#{endLine}")
- public List getListHtml(@Param("listTableName") String tableName,
- @Param("startLine") int startLine, @Param("endLine") int endLine);
-
- @Insert("insert ${postUrlsTableName} (`url`,`extractedTime`) values (#{urlPost},#{extractedTime})")
- public int addPostUrls(@Param("postUrlsTableName") String tableName,
- @Param("urlPost") String urlPost,
- @Param("extractedTime") String extractedTime);
-
- @Update("update ${listTableName} set extracted=#{extracted} where pageMd5=#{pageMd5}")
- public int timestamp(@Param("listTableName") String listTableName,
- @Param("pageMd5") String pageMd5,
- @Param("extracted") String extracted);
-
- @Select("select * from ${table} where id > #{startId} LIMIT #{batchSize}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("batchSize") int batchSize);
-
- @Update("create table ${table} ${fields}")
- public int createTable(@Param("table") String table, @Param("fields") String fields);
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/PointerDAO.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/PointerDAO.java
deleted file mode 100644
index 49d14af05..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/PointerDAO.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-import org.apache.ibatis.annotations.Param;
-
-public interface PointerDAO {
-
- @Select("select pointer from ${table} where table_name = #{table_name}")
- public int readPointer(@Param("table")String table,@Param("table_name") String tableName);
-
-
- @Insert("insert into ${table} (`table_name`, `pointer`) values (#{table_name}, #{pointer})")
- public int insertPointer(@Param("table") String table, @Param("table_name") String tableName,
- @Param("pointer") Integer pointer);
-
- @Update("update ${table} set pointer=#{pointer} where table_name=#{table_name}")
- public int updatePointer(@Param("table") String table, @Param("table_name") String tableName,
- @Param("pointer") int pointer);
-
-// @Select("select pointer from pointers where table_name = #{table_name}")
-// public String readMode(@Param("table_name") String table);
-//
-// @Insert("insert into pointers (`table_name`, `pointer`) values (#{table_name}, #{pointer})")
-// public int insertMode(@Param("table_name") String table,
-// @Param("pointer") Integer pointer);
-//
-// @Update("update pointers set pointer=#{pointer} where table_name=#{table_name}")
-// public int updateMode(@Param("table_name") String table,
-// @Param("pointer") int pointer);
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/URLDao.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/URLDao.java
deleted file mode 100644
index c90e2d71a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/dao/URLDao.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package net.trustie.webmagic.one.dao;
-
-import net.trustie.webmagic.one.model.URL;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-import java.util.List;
-
-/**
- * Created by gan on 2014/11/25.
- */
-public interface URLDao {
- @Insert("insert ${listTableName} (`url`,`crawledTime`) values (#{urlPost},#{crawledTime})")
- public int add(@Param("urlPost") String urlPost,
- @Param("crawledTime") String crawledTime,
- @Param("listTableName") String listTableName);
-
- @Select("select url from ${listTableName} where `timestamp` is NULL limit #{startLine},#{endLine}")
- public List getUrl(@Param("listTableName") String listTableName,
- @Param("startLine") int startLine, @Param("endLine") int endLine);
-
- @Update("update ${listTableName} set timestamp=#{timestamp} where url=#{url}")
- public int timestamp(@Param("timestamp") String timestamp,
- @Param("url") String url,
- @Param("listTableName") String listTableName);
-
- @Select("select url from target_url")
- public List getAllUrl();
-
- @Select("select * from ${table} where id > #{startId} LIMIT #{batchSize}")
- public List getBatch(@Param("table") String tableName,
- @Param("startId") int startId, @Param("batchSize") int batchSize);
-
- @Insert("insert ${table} (`url`, `extractedTime`) values (#{url}, #{extractedTime})")
- public int addURL(@Param("table") String table,
- @Param("url") String url, @Param("extractedTime") String timestamp);
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java
deleted file mode 100644
index f78ae15b3..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java
+++ /dev/null
@@ -1,134 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-public abstract class AbstractHtml {
- private int id;
- private String url;
- private String html;
- private String crawledTime;
- private String urlMd5 = "";
- private String pageMd5 = "";
- private int statusCode = 200;
- // 页面类型
- private String type = "Detail";
-
- /**
- * @return the type
- */
- public String getType() {
- return type;
- }
-
- /**
- * @param type
- * the type to set
- */
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- * @return the id
- */
- public int getId() {
- return id;
- }
-
- /**
- * @return the url
- */
- public String getUrl() {
- return url;
- }
-
- /**
- * @return the html
- */
- public String getHtml() {
- return html;
- }
-
- /**
- * @return the crawledTime
- */
- public String getCrawledTime() {
- return crawledTime;
- }
-
- /**
- * @return the urlMd5
- */
- public String getUrlMd5() {
- return urlMd5;
- }
-
- /**
- * @return the pageMd5
- */
- public String getPageMd5() {
- return pageMd5;
- }
-
- /**
- * @return the statusCode
- */
- public int getStatusCode() {
- return statusCode;
- }
-
- /**
- * @param id
- * the id to set
- */
- public void setId(int id) {
- this.id = id;
- }
-
- /**
- * @param url
- * the url to set
- */
- public void setUrl(String url) {
- this.url = url;
- }
-
- /**
- * @param html
- * the html to set
- */
- public void setHtml(String html) {
- this.html = html;
- }
-
- /**
- * @param crawledTime
- * the crawledTime to set
- */
- public void setCrawledTime(String crawledTime) {
- this.crawledTime = crawledTime;
- }
-
- /**
- * @param urlMd5
- * the urlMd5 to set
- */
- public void setUrlMd5(String urlMd5) {
- this.urlMd5 = urlMd5;
- }
-
- /**
- * @param pageMd5
- * the pageMd5 to set
- */
- public void setPageMd5(String pageMd5) {
- this.pageMd5 = pageMd5;
- }
-
- /**
- * @param statusCode
- * the statusCode to set
- */
- public void setStatusCode(int statusCode) {
- this.statusCode = statusCode;
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java
deleted file mode 100644
index aaaf5caf2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-/**
- * Created by Administrator on 2014/11/16.
- */
-public class DetailHtml extends AbstractHtml{
- private int history = 0;
-
- /**
- *
- */
- public DetailHtml() {
- }
-
- /**
- * @return the history
- */
- public int getHistory() {
- return history;
- }
-
- /**
- * @param history the history to set
- */
- public void setHistory(int history) {
- this.history = history;
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/ListHtml.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/ListHtml.java
deleted file mode 100644
index e582da0ff..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/ListHtml.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-import com.sun.jna.platform.win32.Sspi.TimeStamp;
-
-/**
- * Created by Administrator on 2014/12/21.
- */
-public class ListHtml extends AbstractHtml{
- private int history;
- private String extracted;
- /**
- *
- */
- public ListHtml() {
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/URL.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/URL.java
deleted file mode 100644
index de1831d53..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/one/model/URL.java
+++ /dev/null
@@ -1,92 +0,0 @@
-package net.trustie.webmagic.one.model;
-
-import java.sql.Timestamp;
-
-
-/**
- * Created by Administrator on 2014/11/27.
- */
-
-public class URL {
- private int id;
- private String url;
- private String type="Detail";
- //private String created
-
- /**
- * @param id
- * @param url
- * @param createdAt
- */
- public URL(int id, String url, Timestamp createdAt) {
- this.id = id;
- this.url = url;
- this.createdAt = createdAt;
- }
-
- /**
- * @return the type
- */
- public String getType() {
- return type;
- }
-
- /**
- * @param type the type to set
- */
- public void setType(String type) {
- this.type = type;
- }
-
- /**
- *
- */
- public URL() {
- }
-
- private Timestamp createdAt;
-
- /**
- * @return the id
- */
- public int getId() {
- return id;
- }
-
- /**
- * @return the url
- */
- public String getUrl() {
- return url;
- }
-
- /**
- * @return the createdAt
- */
- public Timestamp getCreatedAt() {
- return createdAt;
- }
-
- /**
- * @param id the id to set
- */
- public void setId(int id) {
- this.id = id;
- }
-
- /**
- * @param url the url to set
- */
- public void setUrl(String url) {
- this.url = url;
- }
-
- /**
- * @param createdAt the createdAt to set
- */
- public void setCreatedAt(Timestamp createdAt) {
- this.createdAt = createdAt;
- }
-
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/ConsolePipelineOther.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/ConsolePipelineOther.java
deleted file mode 100644
index 1ea94e861..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/ConsolePipelineOther.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class ConsolePipelineOther implements Pipeline {
- @Override
- public void process(ResultItems resultItems, Task task) {
- System.out.println("get page: " + resultItems.getRequest().getUrl());
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter.java
deleted file mode 100644
index 50262b996..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.handler.PatternRequestMatcher;
-import us.codecraft.webmagic.handler.RequestMatcher;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class HtmlSubProcessorPatter extends PatternRequestMatcher implements
- SubPageProcessor {
-
- /**
- * @param pattern
- * url pattern to handle
- */
- public HtmlSubProcessorPatter(String pattern) {
- super(pattern);
- }
-
- @Override
- public MatchOther processPage(Page page) {
- System.out.println("i konwn it!URL_LIST");
- return MatchOther.YES;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter2.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter2.java
deleted file mode 100644
index 81e5f300b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorPatter2.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.handler.PatternProcessor;
-import us.codecraft.webmagic.handler.PatternRequestMatcher;
-import us.codecraft.webmagic.handler.RequestMatcher;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-class HtmlSubProcessorPatter2 extends PatternProcessor {
-
- /**
- * @param pattern
- * url pattern to handle
- */
- public HtmlSubProcessorPatter2(String pattern) {
- super(pattern);
- }
-
- @Override
- public MatchOther processPage(Page page) {
- return null;
- }
-
- @Override
- public MatchOther processResult(ResultItems resultItems, Task task) {
- return null;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorTest2.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorTest2.java
deleted file mode 100644
index 08676cb50..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/oneX/HtmlSubProcessorTest2.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package net.trustie.webmagic.oneX;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.handler.PatternRequestMatcher;
-import us.codecraft.webmagic.handler.SubPageProcessor;
-
-/**
- * Created by Administrator on 2014/11/15.
- */
-public class HtmlSubProcessorTest2 extends PatternRequestMatcher implements
- SubPageProcessor {
-
- /**
- * @param pattern
- * url pattern to handle
- */
- public HtmlSubProcessorTest2(String pattern) {
- super(pattern);
- }
-
- @Override
- public MatchOther processPage(Page page) {
- System.out.println("i got it!URL_POST");
- return MatchOther.YES;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate.java
deleted file mode 100644
index 03461306f..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.model.OOSpider;
-import us.codecraft.webmagic.pipeline.PageModelPipeline;
-
-/**
- * Created by gyiang on 2014/8/19.
- */
-public class AutoUpdate {
-
- // private static Spider s;
- public static void go(Spider spider, int startIndex, int endIndex, int f,
- String purl, String furl, int sleepTime, int threadNum,
- PageModelPipeline pageModelPipeline, Class... pageModels) {
- // StoreConf.StoreXML(spider,threadNum,startIndex,endIndex,f,sleepTime);
- OrderSpider.startSpider(spider, startIndex, endIndex, f, purl, furl,0);
-
- while (true) {
- LoadConf prop = new LoadConf(spider.getSite().getDomain());
- Spider s = OOSpider
- .create(Site.me().setRetryTimes(prop.getRetryTimes())
- .setCycleRetryTimes(prop.getCycleRetryTimes())
- .setSleepTime(prop.getSleepTimeThread())
- .setUserAgent(prop.getUserAgent())
- .setDomain(prop.getDomain()), pageModelPipeline,
- pageModels).setSpawnUrl(prop.isSpawnUrl())
- // .setScheduler(spider.getScheduler())
- .thread(threadNum);
- OrderSpider.startSpider(s, prop.getStartPageIndex(),
- prop.getEndInPageIndex(), prop.getPageF(), purl, furl,0);
- // s= spider;
- try {
- Thread.sleep(prop.getSleepTimeThread());
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate_.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate_.java
deleted file mode 100644
index ad222eeeb..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/AutoUpdate_.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.model.OOSpider;
-import us.codecraft.webmagic.pipeline.PageModelPipeline;
-
-/**
- * Created by gyiang on 2014/8/19.
- */
-public class AutoUpdate_ {
-
- // private static Spider s;
- public static void go(Spider spider, int startIndex, int endIndex, int f,
- String purl, String furl, int sleepTime, int threadNum,
- PageModelPipeline pageModelPipeline, Class... pageModels) {
- OrderSpider.startSpider(spider, startIndex, endIndex, f, purl, furl,0);
- while (true) {
- Spider s = OOSpider
- .create(Site
- .me()
- .setRetryTimes(spider.getSite().getRetryTimes())
- .setCycleRetryTimes(
- spider.getSite().getCycleRetryTimes())
- .setSleepTime(spider.getSite().getSleepTime())
- .setUserAgent(spider.getSite().getUserAgent()),
- pageModelPipeline, pageModels)
- .setSpawnUrl(spider.isSpawnUrl())
- // .setScheduler(spider.getScheduler())
- .thread(threadNum);
- OrderSpider.startSpider(s, startIndex, endIndex, f, purl, furl,0);
- // s= spider;
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DBPipeline.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DBPipeline.java
deleted file mode 100644
index 821fd3715..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DBPipeline.java
+++ /dev/null
@@ -1,60 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.pipeline.Pipeline;
-
-public abstract class DBPipeline implements Pipeline {
- private String listHtmlTableName = "";
- private String detailHtmlTableName = "";
- private String errorPageTableName = "";
- private String errorUrlTableName = "";
- /**
- * @return the listHtmlTableName
- */
- public String getListHtmlTableName() {
- return listHtmlTableName;
- }
- /**
- * @return the detailHtmlTableName
- */
- public String getDetailHtmlTableName() {
- return detailHtmlTableName;
- }
- /**
- * @return the errorPageTableName
- */
- public String getErrorPageTableName() {
- return errorPageTableName;
- }
- /**
- * @return the errorUrlTableName
- */
- public String getErrorUrlTableName() {
- return errorUrlTableName;
- }
- /**
- * @param listHtmlTableName the listHtmlTableName to set
- */
- public void setListHtmlTableName(String listHtmlTableName) {
- this.listHtmlTableName = listHtmlTableName;
- }
- /**
- * @param detailHtmlTableName the detailHtmlTableName to set
- */
- public void setDetailHtmlTableName(String detailHtmlTableName) {
- this.detailHtmlTableName = detailHtmlTableName;
- }
- /**
- * @param errorPageTableName the errorPageTableName to set
- */
- public void setErrorPageTableName(String errorPageTableName) {
- this.errorPageTableName = errorPageTableName;
- }
- /**
- * @param errorUrlTableName the errorUrlTableName to set
- */
- public void setErrorUrlTableName(String errorUrlTableName) {
- this.errorUrlTableName = errorUrlTableName;
- }
-
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DateFormat.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DateFormat.java
deleted file mode 100644
index f9e5c02c6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/DateFormat.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.sql.Timestamp;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Locale;
-
-/**
- * Created by Administrator on 2014/10/2.
- */
-public class DateFormat {
-
- public static String formatFrom(String formatString, String format) {
-
- SimpleDateFormat bartDateFormat = new SimpleDateFormat(format,
- Locale.ENGLISH);
- SimpleDateFormat bartDateFormat2 = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss", Locale.ENGLISH);
-
- String date = "2000-01-01 00:00:00";
- try {
- date = bartDateFormat2.format(bartDateFormat.parse(formatString))
- .toString();
- } catch (ParseException e) {
- e.printStackTrace();
- }
- return date;
- }
-
- public static String formatFrom2(String strTime) {
-
- if (strTime.contains("昨天"))
- strTime = "1天前";
- String time = "2000-01-01 00:00:00";
- int vanishTime = Integer.parseInt(strTime.replaceAll("[^\\d]", ""));
- String[] ways = strTime.split("\\d+");
-
- String unit = ways[1];
- if (unit.contains("秒前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.SECOND)).toString();
- } else if (unit.contains("分钟前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.MINUTE)).toString();
- } else if (unit.contains("小时前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.HOUR)).toString();
- } else if (unit.contains("天前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.DAY)).toString();
- } else if (unit.contains("个月前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.MONTH)).toString();
- } else if (unit.contains("年前")) {
- time = (new Timestamp(System.currentTimeMillis() - vanishTime
- * VanishTime.YEAR)).toString();
- } else {
- return null;
- }
- return time;
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/FileReader.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/FileReader.java
deleted file mode 100644
index 6d6baa65e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/FileReader.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.util.Scanner;
-
-public class FileReader {
- public static String readFile(String path) {
- File file = new File(path);
- String rt = "";
- try {
- Scanner sc = new Scanner(file);
- while(sc.hasNextLine()){
- rt += sc.nextLine();
- rt+=" ";
- }
- } catch (FileNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return rt;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetJSON.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetJSON.java
deleted file mode 100644
index ed4c753b6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetJSON.java
+++ /dev/null
@@ -1,76 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.protocol.HTTP;
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-/**
- * Created by Administrator on 2014/7/31.
- */
-public class GetJSON {
- public static void main(String[] args) throws JSONException, IOException {
-
- JSONObject json = new JSONObject();
- json.put("period", "7");
- json.put(
- "__RequestVerificationToken",
- "G7lAAjIljV3yoksJNBfesXVbBm9M4ZF0nS8yjs0i2QYkiay4O7ybpgVD07NOiXhUkCURkwW0wLdAIiOagn-nAoJJxjs0I-oYRyMXX8Z6pixZ7DNRWhrfd49RTTmEPV8APA0HDg2");
- HttpPost httpPost = new HttpPost(
- "http://kinectmultipoint.codeplex.com/stats/getActivity");
- StringEntity entity = new StringEntity(json.toString(), HTTP.UTF_8);
- entity.setContentType("application/json");
- httpPost.setEntity(entity);
- // HttpClient client = new DefaultHttpClient();
- HttpClient client = null;
- HttpResponse response = client.execute(httpPost);
- System.out.println(response.toString());
-
- /*
- * List params = new ArrayList();
- * params.add(new BasicNameValuePair("period", "7")); params.add(new
- * BasicNameValuePair("__RequestVerificationToke...",
- * "G7lAAjIljV3yoksJNBfesXVbBm9M4ZF0nS8yjs0i2QYkiay4O7ybpgVD07NOiXhUkCURkwW0wLdAIiOagn-nAoJJxjs0I-oYRyMXX8Z6pixZ7DNRWhrfd49RTTmEPV8APA0HDg2"
- * )); //要传递的参数. String url =
- * "http://kinectmultipoint.codeplex.com/stats/getActivity?" +
- * URLEncodedUtils.format(params, HTTP.UTF_8); //拼接路径字符串将参数包含进去 json =
- * get(url);
- */
- System.out.println(json.get("Visits"));
-
- }
-
- public static JSONObject get(String url) {
-
- // HttpClient client = new DefaultHttpClient();
- HttpClient client = null;
- HttpGet get = new HttpGet(url);
- JSONObject json = null;
- try {
- HttpResponse res = client.execute(get);
- if (res.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- HttpEntity entity = res.getEntity();
- json = new JSONObject(new JSONTokener(new InputStreamReader(
- entity.getContent(), HTTP.UTF_8)));
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
-
- } finally {
- // 关闭连接 ,释放资源
- client.getConnectionManager().shutdown();
- }
- return json;
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetUrlsProjSF.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetUrlsProjSF.java
deleted file mode 100644
index a113a57f7..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/GetUrlsProjSF.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import org.apache.log4j.Logger;
-
-import java.sql.*;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Created by Administrator on 2014/9/30.
- */
-public class GetUrlsProjSF {
- public static List get() throws SQLException,
- ClassNotFoundException, IllegalAccessException,
- InstantiationException {
- Logger log4j = Logger.getLogger(GetUrlsProjSF.class);
- List l = new ArrayList();
- String sql2 = "select url from proj_sourceforge_url";
- Class.forName("com.mysql.jdbc.Driver").newInstance();
- Connection conn = DriverManager.getConnection(
- "jdbc:mysql://192.168.80.104:3306/oss_tree_fix", "influx",
- "influx1234");
- // Statement stmt = conn.createStatement();
- PreparedStatement ps = conn.prepareStatement(sql2);
- ResultSet rs = ps.executeQuery();
- int i = 0;
- while (rs.next()) {
- String t = rs.getString("url");
- log4j.info("added!#" + t);
- i++;
- l.add(t);
- /*
- * if (i==100) break;
- */
- }
- log4j.info("total urls:" + i);
- log4j.error("well down");
- return l;
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LoadConf.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LoadConf.java
deleted file mode 100644
index 6a9429516..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LoadConf.java
+++ /dev/null
@@ -1,189 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import org.apache.log4j.Logger;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.InvalidPropertiesFormatException;
-import java.util.Properties;
-
-public class LoadConf {
- private int sleepTimeThread = 5000;
- private int retryTimes = 0;
- private int cycleRetryTimes = 0;
- private int timeOut = 5000;
- private String userAgent = "UserAgent.Browser";
- private String charset;
- private int threadNum = 5;
- private boolean isSpawnUrl = true;
- private int startPageIndex = 1;
- private int endInPageIndex = 6;
- private int pageF = 1;
- private int sleepTimeSpider = 36000;
- private String domain;
- private Properties prop = new Properties();
- Logger log4j = Logger.getLogger(LoadConf.class);
-
- public LoadConf() {
- try {
- FileInputStream fis = null;
- fis = new FileInputStream("spider.conf.xml");
- prop.loadFromXML(fis);
- prop.list(System.out);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
-
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- public LoadConf(String confPath) {
- System.out.println(new File(".").getAbsolutePath());
- confPath = "conf/" + "conf." + confPath + ".xml";
- System.err.println(new File(confPath).getAbsolutePath());
- try {
- FileInputStream fis = null;
- fis = new FileInputStream(confPath);
- prop.loadFromXML(fis);
- prop.list(System.out);
- } catch (FileNotFoundException e) {
- log4j.error("FileNotFoundException & loading default config xml! & no default config file");
- e.printStackTrace();
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- private void config() {
- this.sleepTimeThread = Integer.parseInt(prop
- .getProperty("sleepTimeThread"));
- this.timeOut = Integer.parseInt(prop.getProperty("timeOut"));
- this.retryTimes = Integer.parseInt(prop.getProperty("retryTimes"));
- this.cycleRetryTimes = Integer.parseInt(prop
- .getProperty("cycleRetryTimes"));
- this.threadNum = Integer.parseInt(prop.getProperty("threadNum"));
- this.startPageIndex = Integer.parseInt(prop
- .getProperty("startPageIndex"));
- this.endInPageIndex = Integer.parseInt(prop
- .getProperty("endInPageIndex"));
- this.pageF = Integer.parseInt(prop.getProperty("pageF"));
- this.sleepTimeSpider = Integer.parseInt(prop
- .getProperty("sleepTimeSpider"));
- this.isSpawnUrl = Boolean.parseBoolean(prop.getProperty("isSpawnUrl"));
- this.userAgent = prop.getProperty("userAgent");
- this.domain = prop.getProperty("domain");
- }
-
- public int getSleepTimeThread() {
- return sleepTimeThread;
- }
-
- public void setSleepTimeThread(int sleepTimeThread) {
- this.sleepTimeThread = sleepTimeThread;
- }
-
- public int getRetryTimes() {
- return retryTimes;
- }
-
- public void setRetryTimes(int retryTimes) {
- this.retryTimes = retryTimes;
- }
-
- public int getCycleRetryTimes() {
- return cycleRetryTimes;
- }
-
- public void setCycleRetryTimes(int cycleRetryTimes) {
- this.cycleRetryTimes = cycleRetryTimes;
- }
-
- public int getTimeOut() {
- return timeOut;
- }
-
- public void setTimeOut(int timeOut) {
- this.timeOut = timeOut;
- }
-
- public String getUserAgent() {
- return userAgent;
- }
-
- public void setUserAgent(String userAgent) {
- this.userAgent = userAgent;
- }
-
- public String getCharset() {
- return charset;
- }
-
- public void setCharset(String charset) {
- this.charset = charset;
- }
-
- public int getThreadNum() {
- return threadNum;
- }
-
- public void setThreadNum(int threadNum) {
- this.threadNum = threadNum;
- }
-
- public boolean isSpawnUrl() {
- return isSpawnUrl;
- }
-
- public void setSpawnUrl(boolean isSpawnUrl) {
- this.isSpawnUrl = isSpawnUrl;
- }
-
- public int getStartPageIndex() {
- return startPageIndex;
- }
-
- public void setStartPageIndex(int startPageIndex) {
- this.startPageIndex = startPageIndex;
- }
-
- public int getEndInPageIndex() {
- return endInPageIndex;
- }
-
- public void setEndInPageIndex(int endInPageIndex) {
- this.endInPageIndex = endInPageIndex;
- }
-
- public int getPageF() {
- return pageF;
- }
-
- public void setPageF(int pageF) {
- this.pageF = pageF;
- }
-
- public int getSleepTimeSpider() {
- return sleepTimeSpider;
- }
-
- public void setSleepTimeSpider(int sleepTimeSpider) {
- this.sleepTimeSpider = sleepTimeSpider;
- }
-
- public String getDomain() {
- return domain;
- }
-
- public void setDomain(String domain) {
- this.domain = domain;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LongSleepHelper.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LongSleepHelper.java
deleted file mode 100644
index e1f57f422..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/LongSleepHelper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import net.trustie.webmagic.one.Configure;
-
-/*
- * 此类用于设置随机长时间段睡眠
- */
-public class LongSleepHelper {
- private long lastSleep;//上一次长睡眠的结束时间
- private int sleepInterval;//下一次长睡眠间隔期
- private int longSleepTime;//此次长睡眠的时间
-
- private int minlongSleepInterval;
- private int maxlongSleepInterval;
- private int minLongSleepTime;
- private int maxLongSleepTime;
- public LongSleepHelper(Configure conf){
- this.minlongSleepInterval = conf.getMinlongSleepInterval();
- this.maxlongSleepInterval = conf.getMaxlongSleepInterval();
- this.minLongSleepTime = conf.getMinlongSleepTime();
- this.maxLongSleepTime = conf.getMaxlongSleepTime();
- }
- //记录当前时间
- public void startMonitor(){
- lastSleep = System.currentTimeMillis();
- //sleepInterval = MyRandomer.getRandomNum(1*60*60*1000,2*60*60*1000);
- sleepInterval = MyRandomer.getRandomNum(this.minlongSleepInterval,this.maxlongSleepInterval);
- }
- public boolean timeToLongSleep(){
- long now = System.currentTimeMillis();
- if(now >= (lastSleep + sleepInterval)){
- //longSleepTime = MyRandomer.getRandomNum(30*60*1000,2*60*60*1000);
- longSleepTime = MyRandomer.getRandomNum(this.minLongSleepTime,this.maxLongSleepTime);
- lastSleep = now + longSleepTime;
- //sleepInterval = MyRandomer.getRandomNum(1*60*60*1000,2*60*60*1000);
- sleepInterval = MyRandomer.getRandomNum(this.minlongSleepInterval,this.maxlongSleepInterval);
- return true;
- }else
- return false;
- }
-
- public int longsleepTime(){
- return longSleepTime;
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/MyRandomer.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/MyRandomer.java
deleted file mode 100644
index 2339716b6..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/MyRandomer.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.util.Random;
-
-
-//用以产生随机数
-public class MyRandomer {
- private static Random random = new Random(66);
-
- //在[min,max)之间产生一个随机数
- public static int getRandomNum(int min,int max){
- return random.nextInt(max-min)+min;
- }
- //在[0,max)之间产生一个随机数
- public static int getRandomNum(int max){
- return random.nextInt(max);
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/OrderSpider.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/OrderSpider.java
deleted file mode 100644
index d5859600d..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/OrderSpider.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.monitor.SpiderMonitor;
-
-import javax.management.JMException;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-
-import net.trustie.webmagic.one.ListHtmlPipeline;
-
-import org.apache.log4j.Logger;
-
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * Created by gyiang on 2014/8/5.
- */
-public class OrderSpider {
- private static Logger logger = Logger.getLogger(OrderSpider.class);
-
- public static List orderUrl(String purl, String surl, int per,
- int incIndex, int startIndex) {
- List urls = new ArrayList();
- for (int g = startIndex; g < startIndex + per; g++) {
- int pageindex = g + incIndex;
- String url = purl + pageindex + surl;
- urls.add(url);
- }
- return urls;
- }
-
- /*
- * @startIndex起始页
- *
- * @endIndex结束页
- *
- * @per每次生成几个链接
- *
- * @purl 前缀
- *
- * @furl 后缀(p、f是什么意思?) 调用方式: OrderSpider.startSpider(spider, 1, 4,2 ,
- * "http://www.oschina.net/project/list?prefix=&sort=view&p=", "");
- */
- public static void startSpider(Spider spider, int startIndex, int endIndex,
- int per, String purl, String furl, int sleepTimeSpider) {
- for (int i = startIndex; i <= endIndex; i += per) {
- // 采用startUrls ,配合0.5.1支持sourceRegion
- spider.startUrls(orderUrl(purl, furl, per, 0, i));
-
- // 爬虫监控用,不需要
- try {
- MBeanServer mbeanServer = ManagementFactory
- .getPlatformMBeanServer();
- ObjectName objName = new ObjectName("WebMagic" + ":name="
- + spider.getSite().getDomain());
- if (mbeanServer.isRegistered(objName))
- mbeanServer.unregisterMBean(objName);
- spider.setUUID(spider.getSite().getDomain());
- SpiderMonitor.instance().register(spider);
- } catch (JMException e) {
- e.printStackTrace();
- }
- spider.run();
- //sleep
- try {
- logger.info("sleep ");
- Thread.sleep(sleepTimeSpider);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Protector.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Protector.java
deleted file mode 100644
index 71ac16b72..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Protector.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import net.trustie.webmagic.one.Configure;
-import us.codecraft.webmagic.Site;
-
-//此类用于设置加强爬虫性能
-public class Protector {
-
- //为了防止ip被封,对设置的参数进行随机化
- public static void setRandomArgs(Site site,Configure conf){
- //随机设置UA
- site.setUserAgent(UserAgentPool.getUserAgent());
- //随机设置每次请求的间隔
- int time = MyRandomer.getRandomNum(conf.getMinInterval(),
- conf.getMaxInterval());
- conf.setSleepTimeSpider(time);
- //site.setSleepTime(time);
- }
-
- public void main(String args[]){
- while(true){
-
- }
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SimHash.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SimHash.java
deleted file mode 100644
index 0fb849f5a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SimHash.java
+++ /dev/null
@@ -1,197 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.StringTokenizer;
-
-/**
- * Created by Administrator on 2014/11/12.
- */
-
-public class SimHash {
-
- private String tokens;
-
- private BigInteger intSimHash;
-
- private String strSimHash;
-
- private int hashbits = 64;
-
- public SimHash(String tokens) {
- this.tokens = tokens;
- this.intSimHash = this.simHash();
- }
-
- public SimHash(String tokens, int hashbits) {
- this.tokens = tokens;
- this.hashbits = hashbits;
- this.intSimHash = this.simHash();
- }
-
- public BigInteger simHash() {
- int[] v = new int[this.hashbits];
- StringTokenizer stringTokens = new StringTokenizer(this.tokens);
- while (stringTokens.hasMoreTokens()) {
- String temp = stringTokens.nextToken();
- BigInteger t = this.hash(temp);
- for (int i = 0; i < this.hashbits; i++) {
- BigInteger bitmask = new BigInteger("1").shiftLeft(i);
- if (t.and(bitmask).signum() != 0) {
- v[i] += 1;
- } else {
- v[i] -= 1;
- }
- }
- }
- BigInteger fingerprint = new BigInteger("0");
- StringBuffer simHashBuffer = new StringBuffer();
- for (int i = 0; i < this.hashbits; i++) {
- if (v[i] >= 0) {
- fingerprint = fingerprint.add(new BigInteger("1").shiftLeft(i));
- simHashBuffer.append("1");
- } else {
- simHashBuffer.append("0");
- }
- }
- this.strSimHash = simHashBuffer.toString();
- System.out.println(this.strSimHash + " length "
- + this.strSimHash.length());
- return fingerprint;
- }
-
- private BigInteger hash(String source) {
- if (source == null || source.length() == 0) {
- return new BigInteger("0");
- } else {
- char[] sourceArray = source.toCharArray();
- BigInteger x = BigInteger.valueOf(((long) sourceArray[0]) << 7);
- BigInteger m = new BigInteger("1000003");
- BigInteger mask = new BigInteger("2").pow(this.hashbits).subtract(
- new BigInteger("1"));
- for (char item : sourceArray) {
- BigInteger temp = BigInteger.valueOf((long) item);
- x = x.multiply(m).xor(temp).and(mask);
- }
- x = x.xor(new BigInteger(String.valueOf(source.length())));
- if (x.equals(new BigInteger("-1"))) {
- x = new BigInteger("-2");
- }
- return x;
- }
- }
-
- /**
- * 取两个二进制的异或,统计为1的个数,就是海明距离
- *
- * @param other
- * @return
- */
-
- public int hammingDistance(SimHash other) {
-
- BigInteger x = this.intSimHash.xor(other.intSimHash);
- int tot = 0;
-
- // 统计x中二进制位数为1的个数
- // 我们想想,一个二进制数减去1,那么,从最后那个1(包括那个1)后面的数字全都反了,对吧,然后,n&(n-1)就相当于把后面的数字清0,
- // 我们看n能做多少次这样的操作就OK了。
-
- while (x.signum() != 0) {
- tot += 1;
- x = x.and(x.subtract(new BigInteger("1")));
- }
- return tot;
- }
-
- /**
- * calculate Hamming Distance between two strings 二进制怕有错,当成字符串,作一个,比较下结果
- *
- * @author
- * @param str1
- * the 1st string
- * @param str2
- * the 2nd string
- * @return Hamming Distance between str1 and str2
- */
- public int getDistance(String str1, String str2) {
- int distance;
- if (str1.length() != str2.length()) {
- distance = -1;
- } else {
- distance = 0;
- for (int i = 0; i < str1.length(); i++) {
- if (str1.charAt(i) != str2.charAt(i)) {
- distance++;
- }
- }
- }
- return distance;
- }
-
- /**
- * 如果海明距离取3,则分成四块,并得到每一块的bigInteger值 ,作为索引值使用
- *
- * @param simHash
- * @param distance
- * @return
- */
- public List subByDistance(SimHash simHash, int distance) {
- int numEach = this.hashbits / (distance + 1);
- List characters = new ArrayList();
-
- StringBuffer buffer = new StringBuffer();
-
- int k = 0;
- for (int i = 0; i < this.intSimHash.bitLength(); i++) {
- boolean sr = simHash.intSimHash.testBit(i);
-
- if (sr) {
- buffer.append("1");
- } else {
- buffer.append("0");
- }
-
- if ((i + 1) % numEach == 0) {
- BigInteger eachValue = new BigInteger(buffer.toString(), 2);
- System.out.println("----" + eachValue);
- buffer.delete(0, buffer.length());
- characters.add(eachValue);
- }
- }
-
- return characters;
- }
-
- public static void main(String[] args) {
- String s = "This is a test string for testing";
-
- SimHash hash1 = new SimHash(s, 64);
- System.out.println(hash1.intSimHash + " "
- + hash1.intSimHash.bitLength());
-
- hash1.subByDistance(hash1, 3);
-
- System.out.println("\n");
- s = "This is a test string for testing, This is a test string for testing abcdef";
- SimHash hash2 = new SimHash(s, 64);
- System.out.println(hash2.intSimHash + " "
- + hash2.intSimHash.bitCount());
- hash1.subByDistance(hash2, 3);
- s = "This is a test string for testing als";
- SimHash hash3 = new SimHash(s, 64);
- System.out.println(hash3.intSimHash + " "
- + hash3.intSimHash.bitCount());
- hash1.subByDistance(hash3, 3);
- System.out.println("============================");
- int dis = hash1.getDistance(hash1.strSimHash, hash2.strSimHash);
-
- System.out.println(hash1.hammingDistance(hash2) + " " + dis);
-
- int dis2 = hash1.getDistance(hash1.strSimHash, hash3.strSimHash);
-
- System.out.println(hash1.hammingDistance(hash3) + " " + dis2);
-
- }
-}
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StoreConf.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StoreConf.java
deleted file mode 100644
index 63dfddf7b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StoreConf.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import us.codecraft.webmagic.Spider;
-
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Properties;
-
-public class StoreConf {
-
- public static void StoreXML(Spider spider, int threadNum, int startIndex,
- int endIndex, int pageF, int sleepTimeSpider) {
- Properties prop = new Properties();
- prop.setProperty("domain", spider.getSite().getDomain() + "");
- prop.setProperty("retryTimes", spider.getSite().getRetryTimes() + "");
- prop.setProperty("cycleRetryTimes", spider.getSite()
- .getCycleRetryTimes() + "");
- prop.setProperty("timeOut", spider.getSite().getTimeOut() + "");
- prop.setProperty("sleepTimeThread", spider.getSite().getSleepTime()
- + "");
- prop.setProperty("userAgent", spider.getSite().getUserAgent() + "");
- prop.setProperty("threadNum", threadNum + "");
- prop.setProperty("isSpawnUrl", spider.isSpawnUrl() + "");
- prop.setProperty("startPageIndex", startIndex + "");
- prop.setProperty("endInPageIndex", endIndex + "");
- prop.setProperty("pageF", pageF + "");
- prop.setProperty("sleepTimeSpider", sleepTimeSpider + "");
- prop.setProperty("proxy", spider.getSite().getHttpProxyPool() + "");
- FileOutputStream fos;
- try {
- fos = new FileOutputStream("spider.conf.xml");
- prop.storeToXML(fos, "Spider");
- fos.flush();
- fos.close();
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StringUtils.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StringUtils.java
deleted file mode 100644
index d02f538b2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/StringUtils.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Created by Administrator on 2014/7/22.
- */
-public class StringUtils {
-
- public static List getDigit(String text) {
- List digitList = new ArrayList();
- Pattern p = Pattern.compile("(\\d+)");
- Matcher m = p.matcher(text);
- while (m.find()) {
- String find = m.group(1).toString();
- digitList.add(Long.valueOf(find));
- }
- return digitList;
- }
-
- public static String getNumber(String text) {
- Pattern p = Pattern.compile("(\\d+)");
- Matcher m = p.matcher(text);
- if (m.find())
- return m.group(1).toString();
- else
- return "0";
- }
-
- public static Set removeDuplicate(List strs) {
- Set rt = new HashSet();
- for (String str : strs) {
- rt.add(str);
- }
- return rt;
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SubSite.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SubSite.java
deleted file mode 100644
index 93a76fd26..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/SubSite.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package net.trustie.webmagic.utils;
-
-public class SubSite {
- String name;
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public int getStartPageIndex() {
- return startPageIndex;
- }
- public void setStartPageIndex(int startPageIndex) {
- this.startPageIndex = startPageIndex;
- }
- public int getEndInPageIndex() {
- return endInPageIndex;
- }
- public void setEndInPageIndex(int endInPageIndex) {
- this.endInPageIndex = endInPageIndex;
- }
- public String getUrlPost() {
- return urlPost;
- }
- public void setUrlPost(String urlPost) {
- this.urlPost = urlPost;
- }
- public String getFixAllRelativeHrefs() {
- return fixAllRelativeHrefs;
- }
- public void setFixAllRelativeHrefs(String fixAllRelativeHrefs) {
- this.fixAllRelativeHrefs = fixAllRelativeHrefs;
- }
- public String getPrefixUrl() {
- return prefixUrl;
- }
- public void setPrefixUrl(String prefixUrl) {
- this.prefixUrl = prefixUrl;
- }
- public String getPostfixUrl() {
- return postfixUrl;
- }
- public void setPostfixUrl(String postfixUrl) {
- this.postfixUrl = postfixUrl;
- }
- public int getIncrementalPages() {
- return incrementalPages;
- }
- public void setIncrementalPages(int incrementalPages) {
- this.incrementalPages = incrementalPages;
- }
- public int getIncrementSleepTime() {
- return incrementSleepTime;
- }
- public void setIncrementSleepTime(int incrementSleepTime) {
- this.incrementSleepTime = incrementSleepTime;
- }
- public int getMode() {
- return mode;
- }
- public void setMode(int mode) {
- this.mode = mode;
- }
- int startPageIndex;
- int endInPageIndex;
- String urlPost;
- String fixAllRelativeHrefs;
- String prefixUrl;
- String postfixUrl;
- int incrementalPages;
- int incrementSleepTime;
- int mode;
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/TableHelper.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/TableHelper.java
deleted file mode 100644
index bfc786f09..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/TableHelper.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import javax.annotation.Resource;
-
-import org.springframework.stereotype.Component;
-
-import net.trustie.webmagic.one.dao.CreateTableDAO;
-
-@Component("tableHelper")
-public class TableHelper {
- @Resource
- private CreateTableDAO dao;
-
- public boolean isTableExist(String tableName){
- try{
- dao.isTableExist(tableName);
- //System.out.println(i);
- return true;
- }catch(Exception e){
- //e.printStackTrace();
- return false;
- }
-
- }
-
- public void createTable(String table, String fields){
- dao.createTable(table, fields);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgent.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgent.java
deleted file mode 100644
index b006a4763..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgent.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package net.trustie.webmagic.utils;
-
-/**
- * Created by gyiang on 2014/8/2.
- */
-public class UserAgent {
- public static final String Baidu = "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)";
- public static final String Google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
- public static final String Youdao = "Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )";
- public static final String Sogou = "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)";
- public static final String Soso = "Sosospider+(+http://help.soso.com/webspider.htm)";
- public static final String Yahoo = "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)";
- public static final String Jike = "Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)";
- public static final String Sinaiask = "iaskspider/2.0(+http://iask.com/help/help_index.html\")";
- public static final String Msn = "msnbot/1.0 (+http://search.msn.com/msnbot.htm)";
-
- public static final String PicBaidu = "Baiduspider-image";
- public static final String PicGoogle = "Googlebot-Image/1.0";
- public static final String PicSogou = "Sogou Pic Spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)";
-
- public static final String Browser = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36";
- // public static final String
- // BrowserLinux="Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803";
- // public static final String
- // BrowserWindows="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET CLR 1.1.4322; .NET4.0C)";
-
- public static final String Mobile = "Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 930) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537";
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgentPool.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgentPool.java
deleted file mode 100644
index c20e29274..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/UserAgentPool.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.util.Random;
-
-
-
-/*
- * Star Lee, 2015-04-13
- *this class is used to generate a random User Agent
- */
-public class UserAgentPool {
-
- //user agent pool
- private static String[] userAgents = {
-
-
- "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
- "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
- "Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )",
- "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
- "Sosospider+(+http://help.soso.com/webspider.htm)",
- "Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)",
- "Mozilla/5.0 (compatible; JikeSpider; +http://shoulu.jike.com/spider.html)",
- "iaskspider/2.0(+http://iask.com/help/help_index.html\")",
- "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
-
-
- //ie6clients
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; XMPP Tiscali Communicator v.10.0.2; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
-
- //ie7clients
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0 ; .NET CLR 2.0.50215; SL Commerce Client v1.0; Tablet PC 2.0",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506)" ,// Windows Mail on Vista
-
- // ie8clients
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506; Media Center PC 5.0; .NET CLR 1.1.4322)",
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Win64; x64; .NET CLR 2.0.50727; SLCC1; Media Center PC 5.0; .NET CLR 3.0.04506)",
-
- // ie9clients
- "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)",
- "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)",
-
- //ie10clients
- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
- "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
- "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
-
- //ie11clients
- "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
- "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; ASU2JS; rv:11.0) like Gecko", //64bit Win8
- //chrome
- "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.309.0 Safari/532.9",
-
- //chrome8
- "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.558.0 Safari/534.10",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML, like Gecko) Ubuntu/10.10 Chrome/8.1.0.0 Safari/540.0",
-
- //chrome9
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML,like Gecko) Chrome/9.1.0.0 Safari/540.0",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.600.0 Safari/534.14",
-
- //chrome10
- "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.613.0 Chrome/10.0.613.0 Safari/534.15",
-
- //chrome11
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.697.0 Safari/534.24",
-
- //chrome12
- "Mozilla/5.0 (X11; CrOS i686 12.0.742.91) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.93 Safari/534.30",
-
- //chrome13
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1",
-
- //chrome14
- "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.815.0 Safari/535.1",
-
- //chrome29
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36",
-
- //chrome32
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36",
-
- //chrome36
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.143 Safari/537.36",
-
- //chrome33
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36",
-
- //chrome31
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36",
-
- //firefox3
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.14) Gecko/2009090216 Ubuntu/9.04 (jaunty) Firefox/3.0.14",
-
- //firefox4
- "Mozilla/5.0 (X11; Linux x86_64; rv:2.0b4) Gecko/20100818 Firefox/4.0b4",
- "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b9pre) Gecko/20101228 Firefox/4.0b9pre",
-
- //firefox5
- "Mozilla/5.0 (Windows NT 6.1; U; ru; rv:5.0.1.6) Gecko/20110501 Firefox/5.0.1 Firefox/5.0.1",
- "Mozilla/5.0 (X11; U; Linux i586; de; rv:5.0) Gecko/20100101 Firefox/5.0",
-
- //firefox6
- "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0a2) Gecko/20110612 Firefox/6.0a2",
-
- //firefox19
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:19.0) Gecko/20100101 Firefox/19.0",
-
- //firefox20
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:20.0) Gecko/20100101 Firefox/20.0",
-
- //firefox25
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0",
-
- //firefox28
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0",
- //safari
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-us) AppleWebKit/525.28.3 (KHTML, like Gecko) Version/3.2.3 Safari/525.28.3",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-gb) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6",
-
- //safari8
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25",
-
- //safari7
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9) AppleWebKit/537.71 (KHTML, like Gecko) Version/7.0 Safari/537.71",
-
- //safari6
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.28.10 (KHTML, like Gecko) Version/6.0.3 Safari/536.28.10",
-
- //safari5
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-us) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-us) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
-
- //safari4
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-us) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10",
- "Mozilla/5.0 (Windows; U; Windows NT 6.1; es-ES) AppleWebKit/531.22.7 (KHTML, like Gecko) Version/4.0.5 Safari/531.22.7",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-us) AppleWebKit/531.2+ (KHTML, like Gecko) Version/4.0.1 Safari/530.18",
-
- //opera
- "Opera/8.0 (Macintosh; PPC Mac OS X; U; en)",
-
- //opera9
- "Opera/9.52 (Windows NT 5.1; U; en)",
- "Opera/9.20 (Macintosh; Intel Mac OS X; U; en)",
-
- //opera10
- "Opera/9.80 (Windows NT 5.2; U; en) Presto/2.2.15 Version/10.10",
- "Opera/9.80 (Macintosh; Intel Mac OS X; U; en) Presto/2.6.30 Version/10.61",
-
- //opera11
- "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
- "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
-
- //opera12
- "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
- "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
-
- //opera15
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36 OPR/15.0.1147.100",
-
- //opera16
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36 OPR/16.0.1196.73",
-
- //opera17
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36 OPR/17.0.1241.53",
-
- //opera18
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.63",
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 OPR/18.0.1284.68",
- "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49",
-
- //opera19
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36 OPR/19.0.1326.39 (Edition Next)",
- "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36 OPR/19.0.1326.56",
- "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36 OPR/19.0.1326.59",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36 OPR/19.0.1326.63",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36 OPR/19.0.1326.63",
-
- //opera20
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.91 Safari/537.36 OPR/20.0.1387.37 (Edition Next)",
- "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.46 Safari/537.36 OPR/20.0.1387.16 (Edition Developer)",
-
- //opera23
- "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36 OPR/23.0.1522.60",
-
- //opera24
- "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.94 Safari/537.36 OPR/24.0.1558.51 (Edition Next)"
- };
-
-
-
-
- public static String getUserAgent(){
- return userAgents[MyRandomer.getRandomNum(userAgents.length)];
- }
-
- /*public static void main(String args[]){
-
- for(int i = 0; i < 20; i++){
- System.out.println(getUserAgent());
- }
-
- }*/
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Utils.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Utils.java
deleted file mode 100644
index 324e89c7e..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/Utils.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package net.trustie.webmagic.utils;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import net.trustie.webmagic.one.Configure;
-
-public class Utils {
- public static List getProxies(Configure conf) {
- List rt = new ArrayList();
- String[] proxy = new String[2];
- proxy[0] = conf.getProxyIp();
- proxy[1] = conf.getProxyPort();
- rt.add(proxy);
- return rt;
- }
-
- public static String getNow() {
- SimpleDateFormat crawledTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- Date date = new Date();
- // String timestamp = timestampFormat.format(date);
- return crawledTimeFormat.format(date);
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/VanishTime.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/VanishTime.java
deleted file mode 100644
index cd198de43..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/VanishTime.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package net.trustie.webmagic.utils;
-
-public class VanishTime {
- public final static long SECOND = 1000;
- public final static long MINUTE = 60 * SECOND;
- public final static long HOUR = 60 * MINUTE;
- public final static long DAY = 24 * HOUR;
- public final static long MONTH = 30 * DAY;
- public final static long YEAR = 12 * MONTH;
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/xmlToDB.java b/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/xmlToDB.java
deleted file mode 100644
index c14db3011..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/java/net/trustie/webmagic/utils/xmlToDB.java
+++ /dev/null
@@ -1,195 +0,0 @@
-package net.trustie.webmagic.utils;
-
-/**
- * Created by gyiang on 2014/8/1.
- */
-
-/**
- 纯属本人习惯.仅供参考!
- */
-import java.io.File;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Iterator;
-import org.dom4j.Document;
-import org.dom4j.Element;
-import org.dom4j.io.SAXReader;
-
-public class xmlToDB {
-
- /**
- * 给定XML,解析后插入数据库
- *
- * @param filePath
- * 文件的路径名:例如:"F:/creatXml.xml"
- * @throws ClassNotFoundException
- *
- */
-
- public static void importXml(String filePath) {
-
- System.out.println("============开始导入============");
-
- // -------------jdbc代码
- try {
- Class.forName("com.mysql.jdbc.Driver");
- String url = "jdbc:mysql://localhost:3306/influx";
- String user = "root";
- String password = "a1passz";
- Connection con = DriverManager.getConnection(url, user, password);
- // 打开事物
- con.setAutoCommit(false);
- // SQL执行语句
- String exeSql = "";
- // 插入值
- String insertValue = "";
- // 插入的列名
- String insertColumName = "";
- // 更新的SQL语句
- String updateSQL = "";
- // SQL操作语句(用来判断数据库里就没有记录等)
- String optionSql = "";
- // 当前记录的ID值
- String rowID = "";
- // ID的列名
- String idName = "";
- int flag = 0;
-
- // ------------读取XML文件,获得document对象.
- SAXReader reader = new SAXReader();
- reader.setEncoding("Utf-8");
- Document document = null;
- document = reader.read(new File(filePath));
- // -----------获取文档的根节点.
- Element root = document.getRootElement();
-
- // 遍历所有table节点
- for (Iterator rootIter = root.elementIterator("Table"); rootIter
- .hasNext();) {
- Element tableNode = (Element) rootIter.next();
- // 得到table节点的Name属性值,用来判断属于哪一个数据库表
- String tableName = tableNode.attributeValue("Name");
- // 打印数据库表名
- System.out.println("数据库表名" + tableName);
- // 取得table节点下的所有子节点,并且遍历
-
- for (Iterator tableIter = tableNode.elementIterator(); tableIter
- .hasNext();) {
- Element row = (Element) tableIter.next();
- // 得到每一条记录里面的数据
- for (Iterator rowIter = row.elementIterator(); rowIter
- .hasNext();) {
-
- flag++;
-
- Element col = (Element) rowIter.next();
-
- if (flag == 1) {
-
- rowID = col.getText();
- idName = col.getName();
-
- }
-
- // 得到当前节点的名字,用来判断属于数据库中的哪一列
- String columName = col.getName();
- // 得到该节点的字段类型
- String columType = col.attributeValue("type");
-
- // 得到当前节点的值,用来插入数据库中
- // 列值。默认为空字符
- String columValue = "";
- // 判断节点字段的类型,然后转换字段类型
- if (columType.equals("date")) {
- columValue = "to_date(substr('" + col.getText()
- + "',1,10),'yyyy-MM-dd')";
-
- } else {
- columValue = "'" + col.getText() + "'";
- // System.out.println(columType);
- }
-
- // 判断是否为最后一个,如果是则不加','号
- if (rowIter.hasNext()) {
- insertValue += columValue + ",";
- insertColumName += columName + ",";
-
- } else {
- insertValue += columValue;
- insertColumName += columName;
-
- }
- // 判断是否为最后一个,如果是则不加','号
- if (rowIter.hasNext()) {
- updateSQL += columName + "=" + columValue + ",";
- } else {
-
- updateSQL += columName + "=" + columValue;
- }
-
- }
-
- optionSql = "select * from " + tableName + " where "
- + idName + "='" + rowID + "'";
- PreparedStatement optionps = con
- .prepareStatement(optionSql);
- ResultSet opRS = optionps.executeQuery();
-
- System.out.println(optionSql);
-
- // 假如数据库里没有记录则插入
- if (!opRS.next()) {
-
- exeSql = "insert into " + tableName + "("
- + insertColumName + ")" + "values("
- + insertValue + ")";
- PreparedStatement ps = con.prepareStatement(exeSql);
- System.out.println("插入");
- System.out.println(exeSql);
- ps.execute();
- opRS.close();
- optionps.close();
- ps.close();
-
- } else {
- // 假如数据库里有记录则更新
- exeSql = "update " + tableName + " set " + updateSQL
- + " where " + idName + "=" + "'" + rowID + "'";
- PreparedStatement ps = con.prepareStatement(exeSql);
- System.out.println("更新");
- System.out.println(exeSql);
- ps.execute();
- opRS.close();
- optionps.close();
- ps.close();
-
- }
- // 插入完一条记录以后清空记录信息
- exeSql = "";
- insertValue = "";
- insertColumName = "";
- updateSQL = "";
- optionSql = "";
- rowID = "";
- idName = "";
- flag = 0;
-
- }
- }
- con.commit();
- con.close();
- System.out.println("============导入完成============");
-
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- public static void main(String[] args) {
- importXml("C:/ProgramData/MySQL/MySQL Server 5.5/data/webmagic/Badges.xml");
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/main.iml b/crawler/moreSmarterCrawler/fetch_networks/src/main/main.iml
deleted file mode 100644
index 489e48878..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/main.iml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/log4j.xml b/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/log4j.xml
deleted file mode 100644
index 157cc96c0..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,77 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/spring/applicationContext-myBatis.xml b/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/spring/applicationContext-myBatis.xml
deleted file mode 100644
index 2db1141a4..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/spring/applicationContext-myBatis.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/spring/applicationContext.xml b/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/spring/applicationContext.xml
deleted file mode 100644
index 7d83874af..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/spring/applicationContext.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/testDBlogger.xml b/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/testDBlogger.xml
deleted file mode 100644
index bc2391d0b..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/main/resources/testDBlogger.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/CSSPathExtractorTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/CSSPathExtractorTest.java
deleted file mode 100644
index d3d5fb8a2..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/CSSPathExtractorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package net.trustie.webmagic.one.extractor;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Set;
-
-import net.trustie.webmagic.extrator.CSSPathExtractor;
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.XPathExtractor;
-
-import org.jsoup.Jsoup;
-
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-
-public class CSSPathExtractorTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- String urlStr = "http://www.freecode.com/";
- String doc = "";
- try {
- URL url = new URL(urlStr);
- doc = Jsoup.parse(url, 100000).html();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- Html html = new Html(UrlUtils.fixAllRelativeHrefs(doc,
- "http://www.freecode.com/"));
- Extractor extractor = new CSSPathExtractor("div.release h2.release-head>a");
- Set urlSet = extractor.extract(html);
- System.out.println(urlSet.size());
- for (String url : urlSet) {
- System.out.println(url);
- }
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/XPathExtractorTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/XPathExtractorTest.java
deleted file mode 100644
index 5c3701866..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/one/extractor/XPathExtractorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package net.trustie.webmagic.one.extractor;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.Set;
-
-import net.trustie.webmagic.extrator.Extractor;
-import net.trustie.webmagic.extrator.XPathExtractor;
-
-import org.jsoup.Jsoup;
-
-import us.codecraft.webmagic.selector.Html;
-import us.codecraft.webmagic.utils.UrlUtils;
-import us.codecraft.xsoup.Xsoup;
-
-public class XPathExtractorTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- String urlStr = "http://www.freecode.com/";
- String doc="";
- try {
- URL url = new URL(urlStr);
- doc= Jsoup.parse(url, 100000).html();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- Html html = new Html(UrlUtils.fixAllRelativeHrefs(doc,
- "http://www.freecode.com/"));
- Extractor extractor = new XPathExtractor("//div[@class='release']/h2[@class='release-head']/a/@href");
- Set urlSet = extractor.extract(html);
- System.out.println(urlSet.size());
- for (String url : urlSet) {
- System.out.println(url);
- }
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/FileReaderTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/FileReaderTest.java
deleted file mode 100644
index a57dce73a..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/FileReaderTest.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.trustie.webmagic.others;
-
-import net.trustie.webmagic.utils.FileReader;
-
-public class FileReaderTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- System.out.println(FileReader.readFile("./sql/list_html.sql"));
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StaticTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StaticTest.java
deleted file mode 100644
index f3e55f06c..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StaticTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package net.trustie.webmagic.others;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import net.trustie.webmagic.one.Configure;
-import net.trustie.webmagic.one.DetailHtmlProcessor;
-import net.trustie.webmagic.one.DetailHtmlSubProcessor;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.handler.CompositePageProcessor;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.scheduler.SimpleScheduler;
-
-public class StaticTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- Configure conf = new Configure("ossean");
- Set acceptStatCode = new HashSet();
- acceptStatCode.add(200);
- acceptStatCode.add(404);
- acceptStatCode.add(500);
- acceptStatCode.add(503);
- Site site = Site.me().setSleepTime(conf.getSleepTimeThread())
- .setTimeOut(conf.getTimeOut())
- .setRetryTimes(conf.getRetryTimes())
- .setCycleRetryTimes(conf.getCycleRetryTimes())
- .setDomain(conf.getDomain()).setUserAgent(conf.getUserAgent())
- .setAcceptStatCode(acceptStatCode);
- List urlList = new ArrayList();
- urlList.add("http://localhost/open_source_projects?page=1");
- while(true){
- Spider
- .create(new DetailHtmlProcessor(site))
- .addPipeline(new ConsolePipeline()).setScheduler(new SimpleScheduler()).thread(conf.getThreadNum()).startUrls(urlList).run();
- }
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StringTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StringTest.java
deleted file mode 100644
index 035bd4279..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/others/StringTest.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package net.trustie.webmagic.others;
-
-public class StringTest {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- String tmp = "abc";
- System.out.println(tmp.toString());
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/spider/SpiderTet.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/spider/SpiderTet.java
deleted file mode 100644
index ecbfba9cb..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/spider/SpiderTet.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package net.trustie.webmagic.spider;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.pipeline.ConsolePipeline;
-import us.codecraft.webmagic.processor.PageProcessor;
-
-public class SpiderTet {
-
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- List urls = new ArrayList();
- urls.add("http://www.oschina.net/question?catalog=1&show=active&p=1");
- urls.add("http://www.oschina.net/question?catalog=1&show=active&p=2");
- urls.add("http://www.oschina.net/question?catalog=1&show=active&p=3");
- Spider spider = Spider.create(new PageProcessor() {
-
- @Override
- public void process(Page arg0) {
- // TODO Auto-generated method stub
- System.out.println(1);
- }
-
- @Override
- public Site getSite() {
- // TODO Auto-generated method stub
- return null;
- }
- }).addPipeline(new ConsolePipeline()).thread(3).startUrls(urls);
- spider.run();
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/BugInfoDaoTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/BugInfoDaoTest.java
deleted file mode 100644
index 631764739..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/BugInfoDaoTest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package net.trustie.webmagic.test.dao;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-
-import javax.annotation.Resource;
-
-/**
- * User: cairne Date: 13-5-13 Time: 下午8:33
- */
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(locations = { "classpath:/spring/applicationContext-*.xml" })
-public class BugInfoDaoTest {
-
- @Ignore
- @Test
- public void test() {
- /*
- * OpenStackBugInfo bugInfo = new OpenStackBugInfo();
- * bugInfo.setSource("a"); try { final int add =
- * bugInfoDAO.add(bugInfo); System.out.println(add); } catch (Exception
- * e) { e.printStackTrace(); }
- */
- }
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/MybatisTest.java b/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/MybatisTest.java
deleted file mode 100644
index da00e1023..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/java/net/trustie/webmagic/test/dao/MybatisTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package net.trustie.webmagic.test.dao;
-
-import java.util.List;
-
-import javax.annotation.Resource;
-
-import net.trustie.webmagic.one.dao.ListHtmlDao;
-import net.trustie.webmagic.one.model.ListHtml;
-
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-@Component
-public class MybatisTest {
- @Resource
- private ListHtmlDao listHtmlDao;
- public List getBatch(){
- return listHtmlDao.getBatch("freecode_html_list", 0 , 4);
- }
- public static void main(String[] args) {
- // TODO Auto-generated method stub
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
- "spring/applicationContext*.xml");
- MybatisTest mbt = applicationContext.getBean(MybatisTest.class);
- List list= mbt.getBatch();
- for(ListHtml html : list){
- System.out.println(html.toString());
- }
- System.out.println(list.size());
- }
-
-}
diff --git a/crawler/moreSmarterCrawler/fetch_networks/src/test/test.iml b/crawler/moreSmarterCrawler/fetch_networks/src/test/test.iml
deleted file mode 100644
index c2700a337..000000000
--- a/crawler/moreSmarterCrawler/fetch_networks/src/test/test.iml
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/crawler/pyspider_scripts/cnblogs_news.py b/crawler/pyspider_scripts/cnblogs_news.py
deleted file mode 100644
index 42ef09c07..000000000
--- a/crawler/pyspider_scripts/cnblogs_news.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: cnblogs_news
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://news.cnblogs.com/n/page/{}/'
-
- self.guess_num = 100
- self.guess_span = 10
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i < 10:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.news_entry > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'cnblogs_news'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/cnblogs_news_new.py b/crawler/pyspider_scripts/cnblogs_news_new.py
deleted file mode 100644
index 217401810..000000000
--- a/crawler/pyspider_scripts/cnblogs_news_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: cnblogs_news
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://news.cnblogs.com/n/page/{}/'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=1*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://news.cnblogs.com/'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'https://news.cnblogs.com/n/page/1/'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('#pager > a:nth-last-child(2)').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.news_entry > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = CnblogsNews(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class CnblogsNews(Base):
- # 表的名字:
- __tablename__ = 'cnblogs_news_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/cnblogs_q_solved.py b/crawler/pyspider_scripts/cnblogs_q_solved.py
deleted file mode 100644
index 0d739ce66..000000000
--- a/crawler/pyspider_scripts/cnblogs_q_solved.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: cnblogs_q_solved
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://q.cnblogs.com/list/solved?page={}'
-
- self.guess_num = 161
- self.guess_span = 10
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.news_item > h2 > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'cnblogs_q_solved'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/cnblogs_q_solved_new.py b/crawler/pyspider_scripts/cnblogs_q_solved_new.py
deleted file mode 100644
index 54758fbc4..000000000
--- a/crawler/pyspider_scripts/cnblogs_q_solved_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: cnblogs_q_solved
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://q.cnblogs.com/list/solved?page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://q.cnblogs.com/list/solved'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'https://q.cnblogs.com/list/solved?page=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('#pager > a:nth-last-child(2)').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.news_item > h2 > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = CnblogsQSolved(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class CnblogsQSolved(Base):
- # 表的名字:
- __tablename__ = 'cnblogs_q_solved_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/cnblogs_q_unsolved.py b/crawler/pyspider_scripts/cnblogs_q_unsolved.py
deleted file mode 100644
index 69e39675e..000000000
--- a/crawler/pyspider_scripts/cnblogs_q_unsolved.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: cnblogs_q_unsolved
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://q.cnblogs.com/list/unsolved?page={}'
-
- self.guess_num = 161
- self.guess_span = 10
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.news_item > h2 > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'cnblogs_q_unsolved'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/cnblogs_q_unsolved_new.py b/crawler/pyspider_scripts/cnblogs_q_unsolved_new.py
deleted file mode 100644
index 60e6e84ae..000000000
--- a/crawler/pyspider_scripts/cnblogs_q_unsolved_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: cnblogs_q_solved
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://q.cnblogs.com/list/unsolved?page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://q.cnblogs.com/list/unsolved'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'https://q.cnblogs.com/list/unsolved?page=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('#pager > a:nth-last-child(2)').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.news_item > h2 > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = CnblogsQUnsolved(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class CnblogsQUnsolved(Base):
- # 表的名字:
- __tablename__ = 'cnblogs_q_unsolved_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/csdn_ask.py b/crawler/pyspider_scripts/csdn_ask.py
deleted file mode 100644
index 96ae13b23..000000000
--- a/crawler/pyspider_scripts/csdn_ask.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: csdn_ask
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'http://ask.csdn.net/p{}'
-
- self.guess_num = 1000
- self.guess_span = 500
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i < 10:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.questions_detail_con dt a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'csdn_ask'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/csdn_ask_new.py b/crawler/pyspider_scripts/csdn_ask_new.py
deleted file mode 100644
index 803e33140..000000000
--- a/crawler/pyspider_scripts/csdn_ask_new.py
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: csdn_ask
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'http://ask.csdn.net/p{}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'http://ask.csdn.net/'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'http://ask.csdn.net/'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- num_span = response.doc('.page-nav a:last-child').attr.href
- pattern = re.compile(r'http://ask.csdn.net/p(.*)')
- full_num = int(pattern.findall(num_span)[0])
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.questions_detail_con>dl>dt>a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = CsdnAsk(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class CsdnAsk(Base):
- # 表的名字:
- __tablename__ = 'csdn_ask_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/csdn_blog.py b/crawler/pyspider_scripts/csdn_blog.py
deleted file mode 100644
index d73e97990..000000000
--- a/crawler/pyspider_scripts/csdn_blog.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: csdn_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'http://blog.csdn.net/?&page={}'
-
- self.guess_num = 340
- self.guess_span = 10
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.blog_list .tracking-ad > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'csdn_blog'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/csdn_blog_new.py b/crawler/pyspider_scripts/csdn_blog_new.py
deleted file mode 100644
index b9372931c..000000000
--- a/crawler/pyspider_scripts/csdn_blog_new.py
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: csdn_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'http://blog.csdn.net/other/newarticle.html?&page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'http://blog.csdn.net/other/newarticle.html'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'http://blog.csdn.net/other/newarticle.html?&page=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- num_span = response.doc('body div.page_nav > span').text()
- pattern = re.compile(r'共(.*)页')
- full_num = int(pattern.findall(num_span)[0])
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.blog_list .tracking-ad > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = CsdnBlog(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class CsdnBlog(Base):
- # 表的名字:
- __tablename__ = 'csdn_blog_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/csdn_topic_new.py b/crawler/pyspider_scripts/csdn_topic_new.py
deleted file mode 100644
index 96a10bf5a..000000000
--- a/crawler/pyspider_scripts/csdn_topic_new.py
+++ /dev/null
@@ -1,140 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: csdn_topic
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.sites = ['Mobile','CloudComputing','Java','DotNET','WebDevelop','Linux']
- self.base_url = 'http://bbs.csdn.net/forums/{}?page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- for site in self.sites:
- list_url = 'http://bbs.csdn.net/forums/{}'.format(site)
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- self.mode = 1 #确保每次on_start重启时是处在增量模式
- first_page = 'http://bbs.csdn.net/forums/{}?page=1'
- for site in self.sites:
- self.crawl(first_page.format(site), callback=self.get_full, retries=5, age=1*60, headers=self.headers)
-
- def get_full(self, response):
- num_text = next(response.doc('.page_nav > ul > li:nth-last-child(1) > span:nth-child(2)').items()).text()
- pattern = re.compile(r'共(.*)页')
- full_num = int(pattern.findall(num_text)[0])
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('div.content tr')
- for each in targets.items():
- target = each('.title > a').attr.href
- flag = each('td:nth-last-child(2) > span.time').text()
- if target is not None:
- self.crawl(target, callback=self.detail_page, headers=self.headers, itag=flag)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = CsdnTopic(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class CsdnTopic(Base):
- # 表的名字:
- __tablename__ = 'csdn_topic_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/db.py b/crawler/pyspider_scripts/db.py
deleted file mode 100644
index f2257bbe4..000000000
--- a/crawler/pyspider_scripts/db.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.sql import func
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-
-#数据库相关操作
-Base = declarative_base()
-
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'stackoverflow_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255))
- history = Column(String(255))
-
-
-class Pointers(Base):
- __tablename__ = 'pointers'
- id = Column(INTEGER, primary_key=True)
- table_name = Column(String(255))
- pointer = Column(INTEGER)
- created_at = Column(DATETIME)
-
-
-class Url(Base):
- # 表的名字:
- __tablename__ = 'stackoverflow_url'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- timestamp = Column(BIGINT)
- extractedTime = Column(DATETIME)
-
-
-engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/crawler')
-DBSession = sessionmaker(bind=engine)
-Base.metadata.create_all(engine)
-
-
-if __name__ == '__main__':
- # session = DBSession()
- # pointer = session.query(Pointers.pointer).filter(Pointers.table_name=='stackoverflow_url').one()[0]
- # url_num = session.query(func.max(Url.id)).one()[0]
- # url = session.query(Url.url).filter(Url.id==pointer+1).one()[0]
- # session.query(Pointers).filter(Pointers.table_name=='stackoverflow_url').update({Pointers.pointer:pointer+1})
- # session.commit()
-
diff --git a/crawler/pyspider_scripts/iteye_ask.py b/crawler/pyspider_scripts/iteye_ask.py
deleted file mode 100644
index b07d470c4..000000000
--- a/crawler/pyspider_scripts/iteye_ask.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: iteye_ask
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'http://www.iteye.com/problems/hot?page={}'
-
- self.guess_num = 1000
- self.guess_span = 500
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.question-summary .summary > h3 > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'iteye_ask'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/iteye_ask_new.py b/crawler/pyspider_scripts/iteye_ask_new.py
deleted file mode 100644
index 8622ed92c..000000000
--- a/crawler/pyspider_scripts/iteye_ask_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: iteye_ask
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'http://www.iteye.com/ask?page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'http://www.iteye.com/ask'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'http://www.iteye.com/ask?page=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('.pagination > a:nth-last-child(2)').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.question-summary .summary > h3 > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = IteyeAsk(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class IteyeAsk(Base):
- # 表的名字:
- __tablename__ = 'iteye_ask_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/iteye_blog_new.py b/crawler/pyspider_scripts/iteye_blog_new.py
deleted file mode 100644
index 692d9b288..000000000
--- a/crawler/pyspider_scripts/iteye_blog_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: iteye_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'http://www.iteye.com/blogs?page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'http://www.iteye.com/blogs'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'http://www.iteye.com/blogs?page=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('div.pagination > a:nth-last-child(2)').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.content h3 a[title]')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = IteyeBlog(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class IteyeBlog(Base):
- # 表的名字:
- __tablename__ = 'iteye_blog_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/iteye_news.py b/crawler/pyspider_scripts/iteye_news.py
deleted file mode 100644
index ffc5c0c78..000000000
--- a/crawler/pyspider_scripts/iteye_news.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'http://www.iteye.com/news?page={}'
-
- self.guess_num = 750
- self.guess_span = 10
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=5*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.content h3 a[title]')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'iteye_news'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/iteye_news_new.py b/crawler/pyspider_scripts/iteye_news_new.py
deleted file mode 100644
index f28c41173..000000000
--- a/crawler/pyspider_scripts/iteye_news_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: iteye_news
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'http://www.iteye.com/news?page={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'http://www.iteye.com/news'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'http://www.iteye.com/news?page=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('div.pagination > a:nth-last-child(2)').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.content h3 a[title]')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = IteyeNews(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class IteyeNews(Base):
- # 表的名字:
- __tablename__ = 'iteye_news_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/openhub.py b/crawler/pyspider_scripts/openhub.py
deleted file mode 100644
index 7ba2eed07..000000000
--- a/crawler/pyspider_scripts/openhub.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: openhub
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://www.openhub.net/p?page={}&query=&ref=explore_project'
-
- self.guess_num = 1000
- self.guess_span = 1000
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=12*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.well.searchable .title.pull-left > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'openhub'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/openhub_new.py b/crawler/pyspider_scripts/openhub_new.py
deleted file mode 100644
index 9f5a35833..000000000
--- a/crawler/pyspider_scripts/openhub_new.py
+++ /dev/null
@@ -1,135 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: openhub
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://www.openhub.net/p?page={}&query=&sort=new'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=3*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://www.openhub.net/p?page=1&query=&sort=new'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'https://www.openhub.net/p?page=1&query=&sort=new'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('ul.pagination > li:nth-last-child(2) > a').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.well.searchable')
- for each in targets.items():
- target = each('.title.pull-left > a').attr.href
- flag = each('#inner_content > div.stats.pull-left > p:nth-child(1) >a').text() # 内容增量
- self.crawl(target, callback=self.detail_page, headers=self.headers, itag=flag)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Openhub(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Openhub(Base):
- # 表的名字:
- __tablename__ = 'openhub_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/oschina_blog.py b/crawler/pyspider_scripts/oschina_blog.py
deleted file mode 100644
index c237a6801..000000000
--- a/crawler/pyspider_scripts/oschina_blog.py
+++ /dev/null
@@ -1,148 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://www.oschina.net/action/ajax/get_more_recommend_blog?classification=0&p={}'
-
- #希望找到一个比较通用的,对于ajax动态加载页面遍历页数的方法。设定一个很大的页数是一个办法,但是不够灵活。
-
- # 开始时,猜测列表页总页数为499页(经历第一轮的不断爬取后,这个数据会更新成更合理的数据)
- self.guess_num = 500
- # 如果前500页未包括所有的列表页,那么再往下猜100页
- self.guess_span = 100
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=24*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- #targets是列表页中的条目
- targets = response.doc('header > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- # self.not_blank<=self.looked,当self.looked+1==self.guess_num时,可以肯定,guess_num范围内的列表页都爬完了
- # self.bot_blank+1==self.guess_num,说明已经爬完了guess_num范围内的页面,并且没有空的列表页
- # 如果所有列表页都非空,则视为没有爬完全部列表页(guess_num不为总数)
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, age=10*24*60*60, headers=self.headers)
- # 如果not_blank+1的数量不等于guess_num的数量,则有可能是还没爬完guess_num范围,也有可能是已经爬完guess_num范围的页面了但是存在空页
- # 如果是已经爬完guess_num的范围内页面了,并且存在空页的情况,则应该让guess_num等全局属性值复原
- elif(self.looked+1 == self.guess_num):
- # self.guess_num 不应该复原,因为下次on_start方法可以直接使用新的更符合情况的guess_num值
- # self.guess_num = 500
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'oschina_blog'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255))
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/oschina_blog_new.py b/crawler/pyspider_scripts/oschina_blog_new.py
deleted file mode 100644
index 94814badf..000000000
--- a/crawler/pyspider_scripts/oschina_blog_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://www.oschina.net/action/ajax/get_more_recent_blog?classification=0&p={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=30)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://www.oschina.net/action/ajax/get_more_recent_blog?classification=0'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- # def full(self):
- # first_page = 'https://www.oschina.net/action/ajax/get_more_recent_blog?classification=0&p=1'
- # self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- # self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- # def get_full(self, response):
- # full_num = int(response.doc('div.pagination > a:nth-last-child(2)').text())
-
- # for i in range(2, full_num+1):
- # list_url = self.base_url.format(str(i))
- # self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('header > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'oschina_blog_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/oschina_news.py b/crawler/pyspider_scripts/oschina_news.py
deleted file mode 100644
index b083d3f6c..000000000
--- a/crawler/pyspider_scripts/oschina_news.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_blog
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://www.oschina.net/action/ajax/get_more_news_list?newsType=&p={}'
-
- self.guess_num = 500
- self.guess_span = 100
- self.not_blank = 0
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=5*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.main-info .title')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc.text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaBlog(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaBlog(Base):
- # 表的名字:
- __tablename__ = 'oschina_news'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/oschina_project.py b/crawler/pyspider_scripts/oschina_project.py
deleted file mode 100644
index 89a8dba34..000000000
--- a/crawler/pyspider_scripts/oschina_project.py
+++ /dev/null
@@ -1,140 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_project
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://www.oschina.net/project/list?company=0&sort=score&lang=0&recommend=false&p={}'
-
- self.guess_num = 1000
- self.guess_span = 500
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=12*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.box.item .box-aw > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'oschina_project'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/oschina_project_new.py b/crawler/pyspider_scripts/oschina_project_new.py
deleted file mode 100644
index 57e785a38..000000000
--- a/crawler/pyspider_scripts/oschina_project_new.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_project
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-import re
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://www.oschina.net/project/list?sort=time&p={}'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24*60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://www.oschina.net/project/list?sort=time'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- first_page = 'https://www.oschina.net/project/list?sort=time&p=1'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('ul.paging > li:nth-last-child(2) > a').text())
-
- for i in range(2, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.box.item .box-aw > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaProject(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaProject(Base):
- # 表的名字:
- __tablename__ = 'oschina_project_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/oschina_question.py b/crawler/pyspider_scripts/oschina_question.py
deleted file mode 100644
index 3663e5632..000000000
--- a/crawler/pyspider_scripts/oschina_question.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_question
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://www.oschina.net/question/_widgets/_list_content?catalog=1&show=active&p={}'
-
- self.guess_num = 1000
- self.guess_span = 500
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=1*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.box-aw.question_detail .title > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'oschina_question'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/oschina_question_new.py b/crawler/pyspider_scripts/oschina_question_new.py
deleted file mode 100644
index 4bf9c9558..000000000
--- a/crawler/pyspider_scripts/oschina_question_new.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: oschina_question
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://www.oschina.net/question?catalog=1&show=time&p={}'
-
- self.mode = 1 # 默认为增量模式
-
- #全量模式辅助变量
- self.guess_num = 500
- self.guess_span = 500
- self.not_blank = 0 #非空列表页计数,每遇到一个非空列表页,+1
- self.looked = 0 #所有处理过的列表页计数
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://www.oschina.net/question?catalog=1&show=time'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- for i in range(1, self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page_full, retries=0, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page_full(self, response):
- targets = response.doc('.box-aw.question_detail .title > a')
- self.looked += 1 # self.looked:每次调用该函数就会加1,表示出现了一个response,即爬了一个列表页
- if targets.length > 0:
- self.not_blank += 1 # 处理的列表页中,不是空列表页的
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if self.looked == self.guess_num-1 and self.looked == self.not_blank: # 处理过的列表页均非空,说明很可能后面还有列表页
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page_full, retries=0, headers=self.headers)
- elif self.looked == self.guess_num-1:
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.box-aw.question_detail .title > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = OschinaQuestion(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class OschinaQuestion(Base):
- # 表的名字:
- __tablename__ = 'oschina_question_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/sof.py b/crawler/pyspider_scripts/sof.py
deleted file mode 100644
index 2772a3549..000000000
--- a/crawler/pyspider_scripts/sof.py
+++ /dev/null
@@ -1,137 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: stackoverflow
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.sql import func
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表,有就忽略
-
- def on_start(self):
- session = self.DBSession()
- pointer = session.query(Pointers.pointer).filter(Pointers.table_name=='stackoverflow_url').one()[0]
- url_num = session.query(func.max(Url.id)).one()[0]
-
- for i in range(pointer+1, url_num+1):
- url = session.query(Url.url).filter(Url.id==i).one()[0]
- print(url)
- self.crawl(url, callback=self.detail_page, headers=self.headers, retries=0)
-
- # 每增一百条就更新下指针
- if (i-pointer)%100==0 or url_num-i<100:
- session.query(Pointers).filter(Pointers.table_name=='stackoverflow_url').update({Pointers.pointer:i})
- session.commit()
-
- session.close()
-
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- # def update_pointer(self):
- # session = self.DBSession()
- # pointer = session.query(Pointers.pointer).filter(Pointers.table_name=='stackoverflow_url').one()[0]
- # session.query(Pointers).filter(Pointers.table_name=='stackoverflow_url').update({Pointers.pointer:pointer+1})
- # session.commit()
- # session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-
-Base = declarative_base()
-
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'stackoverflow_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255))
- history = Column(String(255))
-
-
-class Pointers(Base):
- __tablename__ = 'pointers'
- id = Column(INTEGER, primary_key=True)
- table_name = Column(String(255))
- pointer = Column(INTEGER)
- created_at = Column(DATETIME)
-
-
-class Url(Base):
- # 表的名字:
- __tablename__ = 'stackoverflow_url'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- timestamp = Column(BIGINT)
- extractedTime = Column(DATETIME)
diff --git a/crawler/pyspider_scripts/sourceforge.py b/crawler/pyspider_scripts/sourceforge.py
deleted file mode 100644
index a6eca08ac..000000000
--- a/crawler/pyspider_scripts/sourceforge.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: sourceforge
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://sourceforge.net/directory/?page={}'
-
- self.guess_num = 1000
- self.guess_span = 1000
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=24 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i == 1:
- self.crawl(list_url, callback=self.index_page, retries=10, age=12*60*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.projects li > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'sourceforge'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/sourceforge_new.py b/crawler/pyspider_scripts/sourceforge_new.py
deleted file mode 100644
index 3afc2ce8d..000000000
--- a/crawler/pyspider_scripts/sourceforge_new.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: sourceforge
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 2*60,
- 3: 3*60,
- 4: 4*60,
- 5: 10*60,
- 6: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update&page={}'
-
- self.mode = 1 # 默认为增量模式
-
- #全量模式辅助变量
- self.guess_num = 500
- self.guess_span = 500
- self.not_blank = 0 #非空列表页计数,每遇到一个非空列表页,+1
- self.looked = 0 #所有处理过的列表页计数
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://influx:influx1234@192.168.80.104:3306/pages')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=60)
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- list_url = 'https://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update'
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, priority=2, headers=self.headers)
-
-
- def full(self):
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- for i in range(1, self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page_full, retries=5, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page_full(self, response):
- targets = response.doc('.projects li > a')
-
- self.looked += 1 # self.looked:每次调用该函数就会加1,表示出现了一个response,即爬了一个列表页
- if targets.length > 0:
- self.not_blank += 1 # 处理的列表页中,不是空列表页的
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if self.looked == self.guess_num-1 and self.looked == self.not_blank: # 处理过的列表页均非空,说明很可能后面还有列表页
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page_full, retries=5, headers=self.headers)
- elif self.looked == self.guess_num-1:
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.projects li > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- @config(priority=1)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Sourceforge(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Sourceforge(Base):
- # 表的名字:
- __tablename__ = 'sourceforge_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/stackoverflow.py b/crawler/pyspider_scripts/stackoverflow.py
deleted file mode 100644
index 1a3afb7f3..000000000
--- a/crawler/pyspider_scripts/stackoverflow.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: stackoverflow
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
- self.base_url = 'https://stackoverflow.com/questions?page={}&sort=newest'
-
- self.guess_num = 1000
- self.guess_span = 1000
-
- # 非空列表页计数,每遇到一个非空列表页,+1
- self.not_blank = 0
- # 所有处理过的列表页计数
- self.looked = 0
-
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=1 * 60)
- def on_start(self):
- for i in range(1,self.guess_num):
- list_url = self.base_url.format(str(i))
- # 第一页需要更加频繁地进行爬取
- if i < 10:
- self.crawl(list_url, callback=self.index_page, retries=10, age=30*60, headers=self.headers)
- else:
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
-
- targets = response.doc('.summary h3 > a')
-
- # 所有页计数,无论空不空,都+1
- self.looked += 1
-
- # 非空页判定
- if(targets.length > 0):
- self.not_blank += 1
-
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
- if(self.not_blank+1 == self.guess_num):
- tmp = self.guess_num
- self.guess_num += self.guess_span
- for i in range(tmp,self.guess_num):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=10, headers=self.headers)
-
- elif(self.looked+1 == self.guess_num):
- self.not_blank = 0
- self.looked = 0
- else:
- pass
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- # "title": response.doc('.blog-content .title').remove('span').text(), #把标题中的“原”,“荐”等无关信息删除掉
- # "html": response.doc('html').text(),
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMD5": self.get_md5(response.url.encode()),
- "pageMD5":self.get_md5(response.text.encode()),
- }
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMD5=result['urlMD5'],crawledTime=result['crawledTime'],
- pageMD5=result['pageMD5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'stackoverflow'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMD5 = Column(String(255))
- pageMD5 = Column(String(255),)
- history = Column(String(255))
-
diff --git a/crawler/pyspider_scripts/stackoverflow_new.py b/crawler/pyspider_scripts/stackoverflow_new.py
deleted file mode 100644
index 557aeea8d..000000000
--- a/crawler/pyspider_scripts/stackoverflow_new.py
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-# Created on 2017-03-20 10:08:31
-# Project: stackoverflow
-
-from pyspider.libs.base_handler import *
-
-import hashlib
-from datetime import *
-
-from sqlalchemy import Column, String, create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.dialects.mysql import \
- BIGINT, BINARY, BIT, BLOB, BOOLEAN, CHAR, DATE, \
- DATETIME, DECIMAL, DECIMAL, DOUBLE, ENUM, FLOAT, INTEGER, \
- LONGBLOB, LONGTEXT, MEDIUMBLOB, MEDIUMINT, MEDIUMTEXT, NCHAR, \
- NUMERIC, NVARCHAR, REAL, SET, SMALLINT, TEXT, TIME, TIMESTAMP, \
- TINYBLOB, TINYINT, TINYTEXT, VARBINARY, VARCHAR, YEAR
-
-class Handler(BaseHandler):
- crawl_config = {
- 'itag':'v1'
- }
-
- retry_delay = {
- 0: 0,
- 1: 60,
- 2: 5*60,
- 3: 10*60,
- 4: 30*60,
- '': 60*60
- }
-
- def __init__(self):
- self.headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
- '(KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'}
-
- self.base_url = 'https://stackoverflow.com/questions?page={}&sort=newest'
-
- self.mode = 1 # 默认为增量模式
-
- # 数据库相关
- self.engine = create_engine('mysql+mysqlconnector://root:root@localhost:3306/ossean')
- self.DBSession = sessionmaker(bind=self.engine)
- Base.metadata.create_all(self.engine) #自建数据库表
-
- @every(minutes=5) #每5分钟增量爬一次
- def on_start(self):
- if self.mode == 1:
- self.increment()
- else:
- self.full()
-
-
- def increment(self):
- for i in range(1, 5):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=5, age=1*60, headers=self.headers)
-
-
- def full(self):
- first_page = 'https://stackoverflow.com/questions'
- self.crawl(first_page, callback=self.get_full, retries=5, age=1*60, headers=self.headers)
- self.mode = 1 #确保每次on_start重启时是处在增量模式
-
- def get_full(self, response):
- full_num = int(response.doc('#mainbar > div.pager.fl > a:nth-child(7) > span').text()) #列表页总数
-
- for i in range(5, full_num+1):
- list_url = self.base_url.format(str(i))
- self.crawl(list_url, callback=self.index_page, retries=0, headers=self.headers)
-
-
- @config(age=10 * 24 * 60 * 60)
- def index_page(self, response):
- targets = response.doc('.summary h3 > a')
- for each in targets.items():
- self.crawl(each.attr.href, callback=self.detail_page, headers=self.headers)
-
-
- @config(priority=2)
- def detail_page(self, response):
- return {
- "url": response.url,
- "html": response.text,
- "crawledTime": datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
- "urlMd5": self.get_md5(response.url.encode()),
- "pageMd5":self.get_md5(response.text.encode()),
- }
-
-
- def on_result(self,result):
- if not result or not str(result['url']): #如果未返回结果或者返回结果的url字段为空,则不做处理
- return
-
- session = self.DBSession() #存库所需
-
- #将result格式化成sqlalchemy能够接受的数据类型(字典转列表)
- data = Table(url=result['url'],html=result['html'],
- urlMd5=result['urlMd5'],crawledTime=result['crawledTime'],
- pageMd5=result['pageMd5']
- )
-
- #存库
- session.merge(data)
- session.commit()
- session.close()
-
- def get_md5(self,data):
- m = hashlib.md5()
- m.update(data)
- return m.hexdigest()
-
-#数据库相关操作
-Base = declarative_base()
-
-class Table(Base):
- # 表的名字:
- __tablename__ = 'stackoverflow_html_detail'
-
- # 表的结构:
- id = Column(INTEGER, primary_key=True)
- url = Column(String(255), nullable=False)
- html = Column(MEDIUMTEXT)
- crawledTime = Column(DATETIME)
- urlMd5 = Column(String(255))
- pageMd5 = Column(String(255))
- history = Column(String(255))
-
-
diff --git a/crawler/pyspider_scripts/test.py b/crawler/pyspider_scripts/test.py
deleted file mode 100644
index e0874a723..000000000
--- a/crawler/pyspider_scripts/test.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-
-sites = ['Mobile','CloudComputing','Java','DotNET','WebDevelop','Linux']
-
-base_url = 'http://bbs.csdn.net/forums/{}?page={}'
-
-for site in sites:
- url = base_url.format(site,1)
- print(url)
\ No newline at end of file
diff --git a/datamonitor/.DS_Store b/datamonitor/.DS_Store
deleted file mode 100644
index 042061006..000000000
Binary files a/datamonitor/.DS_Store and /dev/null differ
diff --git a/datamonitor/.classpath b/datamonitor/.classpath
deleted file mode 100644
index 200109268..000000000
--- a/datamonitor/.classpath
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/datamonitor/.project b/datamonitor/.project
deleted file mode 100644
index 9d0e94bc3..000000000
--- a/datamonitor/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
- datamonitor
-
-
-
-
-
- org.eclipse.jdt.core.javabuilder
-
-
-
-
- org.eclipse.m2e.core.maven2Builder
-
-
-
-
-
- org.eclipse.jdt.core.javanature
- org.eclipse.m2e.core.maven2Nature
-
-
diff --git a/datamonitor/.settings/org.eclipse.core.resources.prefs b/datamonitor/.settings/org.eclipse.core.resources.prefs
deleted file mode 100644
index f9fe34593..000000000
--- a/datamonitor/.settings/org.eclipse.core.resources.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-eclipse.preferences.version=1
-encoding//src/main/java=UTF-8
-encoding//src/test/java=UTF-8
-encoding/=UTF-8
diff --git a/datamonitor/.settings/org.eclipse.jdt.core.prefs b/datamonitor/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index abec6ca38..000000000
--- a/datamonitor/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
-org.eclipse.jdt.core.compiler.compliance=1.5
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.5
diff --git a/datamonitor/.settings/org.eclipse.m2e.core.prefs b/datamonitor/.settings/org.eclipse.m2e.core.prefs
deleted file mode 100644
index f897a7f1c..000000000
--- a/datamonitor/.settings/org.eclipse.m2e.core.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-activeProfiles=
-eclipse.preferences.version=1
-resolveWorkspaceProjects=true
-version=1
diff --git a/datamonitor/bin/.DS_Store b/datamonitor/bin/.DS_Store
deleted file mode 100644
index 79a61f77b..000000000
Binary files a/datamonitor/bin/.DS_Store and /dev/null differ
diff --git a/datamonitor/bin/resource/applicationContext-mybatis.xml b/datamonitor/bin/resource/applicationContext-mybatis.xml
deleted file mode 100644
index bdf2b803c..000000000
--- a/datamonitor/bin/resource/applicationContext-mybatis.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/datamonitor/bin/resource/applicationContext.xml b/datamonitor/bin/resource/applicationContext.xml
deleted file mode 100644
index 02787952a..000000000
--- a/datamonitor/bin/resource/applicationContext.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/datamonitor/bin/resource/log4j.xml b/datamonitor/bin/resource/log4j.xml
deleted file mode 100644
index 157cc96c0..000000000
--- a/datamonitor/bin/resource/log4j.xml
+++ /dev/null
@@ -1,77 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/datamonitor/bin/start.sh b/datamonitor/bin/start.sh
deleted file mode 100644
index 53af2fb02..000000000
--- a/datamonitor/bin/start.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-SITE='datamonitor'
-
-find ./target/classes -name "*.xml"|xargs rm -f
-
-tmp='./bin/resources'
-tmp='./target/classes':$tmp
-tmp='./target/datamonitor-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-
-CLASSPATH=$tmp:$CLASSPATH
-
-JAVA_OPTS="-Xms256m -Xmx256m -Xmn128m"
-
-echo $CLASSPATH
-
-java $JAVA_OPTS -DlogFilePath=${SITE} -classpath $CLASSPATH com.ossean.datamonitor.Monitor 2>&1 &
\ No newline at end of file
diff --git a/datamonitor/config/crawlTables.txt b/datamonitor/config/crawlTables.txt
deleted file mode 100644
index d97b35de9..000000000
--- a/datamonitor/config/crawlTables.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-csdn_ask topic crawledTime
-csdn_blogs blog crawledTime
-csdn_topics blog crawledTime
-cnblog_news news crawledTime
-cnblog_question topic crawledTime
-dewen_question topic crawledTime
-freecode_projects project crawledTime
-iteye_ask topic crawledTime
-openhub_project project crawledTime
-oschina_project project crawledTime
-oschina_question topic crawledTime
-sourceforge_project project crawledTime
-stackoverflow_q topic crawledTime
diff --git a/datamonitor/config/extractTables.txt b/datamonitor/config/extractTables.txt
deleted file mode 100644
index bb4d56877..000000000
--- a/datamonitor/config/extractTables.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-csdn_ask topic extractTime
-csdn_blogs blog extractTime
-csdn_topics blog extractTime
-cnblog_news news extractTime
-cnblog_question topic extractTime
-dewen_question topic extractTime
-freecode_projects project extractTime
-iteye_ask topic extractTime
-openhub_project project extractTime
-oschina_project project extractTime
-oschina_question topic extractTime
-sourceforge_project project extractTime
-stackoverflow_q topic extractTime
\ No newline at end of file
diff --git a/datamonitor/pom.xml b/datamonitor/pom.xml
deleted file mode 100644
index 2b587be91..000000000
--- a/datamonitor/pom.xml
+++ /dev/null
@@ -1,90 +0,0 @@
-
- 4.0.0
-
- com.ossean
- datamonitor
- 0.0.1-SNAPSHOT
- jar
-
- datamonitor
- http://maven.apache.org
-
-
- UTF-8
-
-
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
- 1.7
- 1.7
-
-
-
- maven-assembly-plugin
- 2.5.1
-
-
- src/main/assembly/assembly.xml
-
-
-
-
- make-assembly
- package
-
- single
-
-
-
-
-
-
-
-
-
-
- junit
- junit
- 3.8.1
- test
-
-
-
- org.mybatis
- mybatis
- 3.1.1
-
-
-
- commons-dbcp
- commons-dbcp
- 1.3
-
-
- mysql
- mysql-connector-java
- 5.1.18
-
-
- org.springframework
- spring-context
- 4.1.4.RELEASE
-
-
- org.mybatis
- mybatis-spring
- 1.1.1
-
-
-
- org.slf4j
- slf4j-log4j12
- 1.7.7
-
-
-
diff --git a/datamonitor/src/.DS_Store b/datamonitor/src/.DS_Store
deleted file mode 100644
index 708d5be84..000000000
Binary files a/datamonitor/src/.DS_Store and /dev/null differ
diff --git a/datamonitor/src/main/assembly/assembly.xml b/datamonitor/src/main/assembly/assembly.xml
deleted file mode 100644
index baf8ba912..000000000
--- a/datamonitor/src/main/assembly/assembly.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
- jar-with-dependencies-without-resources
-
- dir
-
- false
-
-
- /
- false
- false
- runtime
-
-
- /
- false
- false
- system
-
-
-
\ No newline at end of file
diff --git a/datamonitor/src/main/java/com/ossean/dao/DestSource.java b/datamonitor/src/main/java/com/ossean/dao/DestSource.java
deleted file mode 100644
index 6f3102993..000000000
--- a/datamonitor/src/main/java/com/ossean/dao/DestSource.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package com.ossean.dao;
-
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-import com.ossean.datamonitor.DesTable;
-
-public interface DestSource {
-
- @Select("select count(*) from ${table} where website = #{website}")
- public int checkItem(@Param("table") String table ,@Param("website")String website);
-
- @Insert("insert into ${table}"+"(`website`)"+"values(#{website})")
- public void addItem(@Param("table") String table, @Param("website") String website);
-
- @Update("update ${table} set category=#{model.category},day_extractor=#{model.day_extractor},week_extractor=#{model.week_extractor},month_extractor=#{model.month_extractor} where website=#{model.website}")
- public void updateExtractorItem(@Param("table") String table, @Param("model") DesTable model);
-
- @Update("update ${table} set category=#{model.category},day_crawler=#{model.day_crawler},week_crawler=#{model.week_crawler},month_crawler=#{model.month_crawler} where website=#{model.website}")
- public void updateCrawlerItem(@Param("table") String table, @Param("model") DesTable model);
-
-}
diff --git a/datamonitor/src/main/java/com/ossean/datamonitor/App.java b/datamonitor/src/main/java/com/ossean/datamonitor/App.java
deleted file mode 100644
index 52660e516..000000000
--- a/datamonitor/src/main/java/com/ossean/datamonitor/App.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package com.ossean.datamonitor;
-
-/**
- * Hello world!
- *
- */
-public class App
-{
- public static void main( String[] args )
- {
- System.out.println( "Hello World!" );
- }
-}
diff --git a/datamonitor/src/main/java/com/ossean/datamonitor/DesTable.java b/datamonitor/src/main/java/com/ossean/datamonitor/DesTable.java
deleted file mode 100644
index 61a32fb1d..000000000
--- a/datamonitor/src/main/java/com/ossean/datamonitor/DesTable.java
+++ /dev/null
@@ -1,107 +0,0 @@
-package com.ossean.datamonitor;
-
-import java.util.Date;
-
-public class DesTable {
- private String website;
- private String category;
- private int day_crawler;
- private int week_crawler;
- private int month_crawler;
- private int day_extractor;
- private int week_extractor;
- private int month_extractor;
- private int flow_num;
- private int match_num;
- private Date new_menos;
- private int week_new_menos;
-
- public DesTable(String website , String catgory){
- this.website = website;
- this.category = catgory;
- }
-
-
-
- public String getWebsite() {
- return website;
- }
- public void setWebsite(String website) {
- this.website = website;
- }
- public String getCatgory() {
- return category;
- }
- public void setCatgory(String catgory) {
- this.category = catgory;
- }
- public int getDay_crawler() {
- return day_crawler;
- }
- public void setDay_crawler(int day_crawler) {
- this.day_crawler = day_crawler;
- }
- public int getWeek_crawler() {
- return week_crawler;
- }
- public void setWeek_crawler(int week_crawler) {
- this.week_crawler = week_crawler;
- }
- public int getMonth_crawler() {
- return month_crawler;
- }
- public void setMonth_crawler(int month_crawler) {
- this.month_crawler = month_crawler;
- }
- public int getDay_extractor() {
- return day_extractor;
- }
- public void setDay_extractor(int day_extractor) {
- this.day_extractor = day_extractor;
- }
- public int getWeek_extractor() {
- return week_extractor;
- }
- public void setWeek_extractor(int week_extractor) {
- this.week_extractor = week_extractor;
- }
- public int getMonth_extractor() {
- return month_extractor;
- }
- public void setMonth_extractor(int month_extractor) {
- this.month_extractor = month_extractor;
- }
- public int getFlow_num() {
- return flow_num;
- }
- public void setFlow_num(int flow_num) {
- this.flow_num = flow_num;
- }
- public int getMatch_num() {
- return match_num;
- }
- public void setMatch_num(int match_num) {
- this.match_num = match_num;
- }
- public Date getNew_menos() {
- return new_menos;
- }
- public void setNew_menos(Date new_menos) {
- this.new_menos = new_menos;
- }
- public int getWeek_new_menos() {
- return week_new_menos;
- }
- public void setWeek_new_menos(int week_new_menos) {
- this.week_new_menos = week_new_menos;
- }
- public int getMonth_new_menos() {
- return month_new_menos;
- }
- public void setMonth_new_menos(int month_new_menos) {
- this.month_new_menos = month_new_menos;
- }
- private int month_new_menos;
-
-
-}
diff --git a/datamonitor/src/main/java/com/ossean/datamonitor/Monitor.java b/datamonitor/src/main/java/com/ossean/datamonitor/Monitor.java
deleted file mode 100644
index 643186a06..000000000
--- a/datamonitor/src/main/java/com/ossean/datamonitor/Monitor.java
+++ /dev/null
@@ -1,153 +0,0 @@
-package com.ossean.datamonitor;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import javax.annotation.Resource;
-
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Repository;
-
-import com.ossean.dao.DestSource;
-import com.ossean.datamonitor.crawlersource.CrawlerSourceData;
-import com.ossean.datamonitor.extractorsource.ExtractorSourceData;
-import com.ossean.util.TxtRead;
-
-
-@Repository("Monitor")
-public class Monitor extends TimerTask {
-
- @Resource
- private CrawlerSourceData crawlerSourceData;
-
- @Resource
- private ExtractorSourceData extractorSourceData;
-
- @Resource
- private DestSource destSource;
-
- private List txt = new ArrayList();
- private List tables = new ArrayList();
- private List category = new ArrayList();
- private List timeCol = new ArrayList();
-
- private String end_time = "23:59:59";
- private String begin_time = "00:00:00";
- private String begin_day;
- private String begin_week;
- private String begin_month;
- private String crawler_tail = "_html_detail";
- private String dest_table="destable";
-
-
-
- @Override
- public void run() {
- // 对抽取环节的监控
- String time_format = "yyyy-MM-dd";
- SimpleDateFormat sdf = new SimpleDateFormat(time_format);
- String end = sdf.format(new Date());
- end = end + " " + end_time;
- begin_day = getTime(-1) + " " + begin_time;
- begin_week = getTime(-7) + " " + begin_time;
- begin_month = getTime(-30) + " " + begin_time;
-
- // 加载extract的配置文件
- getExtractorConfig("extractTables");
- checkItem(tables);
- for(int i = 0 ; i tables){
- for(String website:tables){
- if(destSource.checkItem(dest_table, website)==0){
- destSource.addItem(dest_table, website);
- }
- }
- }
-
- public static String getTime(int days) {
- String time_format = "yyyy-MM-dd";
- SimpleDateFormat sdf = new SimpleDateFormat(time_format);
- Calendar c = Calendar.getInstance();
- c.add(Calendar.DAY_OF_MONTH, days);
- return sdf.format(c.getTime());
- }
-
- public void begin() {
- Timer timer = new Timer();
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- String now = sdf.format(new Date());
- Date firstTime = null;
- try {
- firstTime = sdf.parse(now.substring(0, 10) + " 23:59:59");
- } catch (ParseException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- timer.schedule(this, 0, 1000);
- }
- public static void main(String[] args) {
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
- Monitor m = (Monitor) applicationContext.getBean("Monitor");
-// Main m = new Main();
- m.begin();
- }
-}
diff --git a/datamonitor/src/main/java/com/ossean/datamonitor/crawlersource/CrawlerSourceData.java b/datamonitor/src/main/java/com/ossean/datamonitor/crawlersource/CrawlerSourceData.java
deleted file mode 100644
index 9268c2860..000000000
--- a/datamonitor/src/main/java/com/ossean/datamonitor/crawlersource/CrawlerSourceData.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.ossean.datamonitor.crawlersource;
-
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-
-public interface CrawlerSourceData {
- @Select("select count(*) from ${table} where ${timeCol} >= #{start} and ${timeCol} < #{end}")
- public int selectByTime(@Param("table") String table, @Param("timeCol") String timeCol, @Param("start") String start, @Param("end") String end);
-
-}
diff --git a/datamonitor/src/main/java/com/ossean/datamonitor/extractorsource/ExtractorSourceData.java b/datamonitor/src/main/java/com/ossean/datamonitor/extractorsource/ExtractorSourceData.java
deleted file mode 100644
index 29f473935..000000000
--- a/datamonitor/src/main/java/com/ossean/datamonitor/extractorsource/ExtractorSourceData.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.ossean.datamonitor.extractorsource;
-
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-
-public interface ExtractorSourceData {
- @Select("select count(*) from ${table} where ${timeCol} >= #{start} and ${timeCol} < #{end}")
- public int selectByTime(@Param("table") String table, @Param("timeCol") String timeCol, @Param("start") String start, @Param("end") String end);
-
-}
diff --git a/datamonitor/src/main/java/com/ossean/util/TxtRead.java b/datamonitor/src/main/java/com/ossean/util/TxtRead.java
deleted file mode 100644
index c1284267a..000000000
--- a/datamonitor/src/main/java/com/ossean/util/TxtRead.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package com.ossean.util;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-
-public class TxtRead {
- public static List read(String path){
- List result = new ArrayList();
- InputStream inputStream;
- try {
- inputStream = new FileInputStream(path);
- InputStreamReader reader = new InputStreamReader(inputStream);
- BufferedReader bufferedReader = new BufferedReader(reader);
- String line;
- try {
- while((line = bufferedReader.readLine()) != null){
- result.add(line);
- }
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- try {
- bufferedReader.close();
- reader.close();
- inputStream.close();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- } catch (FileNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- return result;
- }
-}
diff --git a/datamonitor/src/main/resource/applicationContext-mybatis.xml b/datamonitor/src/main/resource/applicationContext-mybatis.xml
deleted file mode 100644
index bb2857719..000000000
--- a/datamonitor/src/main/resource/applicationContext-mybatis.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/datamonitor/src/main/resource/applicationContext.xml b/datamonitor/src/main/resource/applicationContext.xml
deleted file mode 100644
index 3c0295d57..000000000
--- a/datamonitor/src/main/resource/applicationContext.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/datamonitor/src/main/resource/log4j.xml b/datamonitor/src/main/resource/log4j.xml
deleted file mode 100644
index 157cc96c0..000000000
--- a/datamonitor/src/main/resource/log4j.xml
+++ /dev/null
@@ -1,77 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/datamonitor/src/test/java/com/ossean/datamonitor/AppTest.java b/datamonitor/src/test/java/com/ossean/datamonitor/AppTest.java
deleted file mode 100644
index 4ea51feff..000000000
--- a/datamonitor/src/test/java/com/ossean/datamonitor/AppTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package com.ossean.datamonitor;
-
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
-/**
- * Unit test for simple App.
- */
-public class AppTest
- extends TestCase
-{
- /**
- * Create the test case
- *
- * @param testName name of the test case
- */
- public AppTest( String testName )
- {
- super( testName );
- }
-
- /**
- * @return the suite of tests being tested
- */
- public static Test suite()
- {
- return new TestSuite( AppTest.class );
- }
-
- /**
- * Rigourous Test :-)
- */
- public void testApp()
- {
- assertTrue( true );
- }
-}
diff --git a/gather_program/.DS_Store b/gather_program/.DS_Store
deleted file mode 100644
index 89fb3c8b2..000000000
Binary files a/gather_program/.DS_Store and /dev/null differ
diff --git a/gather_program/.gitignore b/gather_program/.gitignore
deleted file mode 100644
index b83d22266..000000000
--- a/gather_program/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/target/
diff --git a/gather_program/.idea/compiler.xml b/gather_program/.idea/compiler.xml
deleted file mode 100644
index a9cc92d28..000000000
--- a/gather_program/.idea/compiler.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/encodings.xml b/gather_program/.idea/encodings.xml
deleted file mode 100644
index b26911bd0..000000000
--- a/gather_program/.idea/encodings.xml
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml b/gather_program/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml
deleted file mode 100644
index 30ff5cb79..000000000
--- a/gather_program/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml b/gather_program/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml
deleted file mode 100644
index 3caee7e54..000000000
--- a/gather_program/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_3.xml b/gather_program/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_3.xml
deleted file mode 100644
index ebd31e46a..000000000
--- a/gather_program/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_3.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml b/gather_program/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml
deleted file mode 100644
index 7b5b3b758..000000000
--- a/gather_program/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml b/gather_program/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml
deleted file mode 100644
index eab40b329..000000000
--- a/gather_program/.idea/libraries/Maven__commons_logging_commons_logging_1_2.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml b/gather_program/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml
deleted file mode 100644
index ab2072126..000000000
--- a/gather_program/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__junit_junit_3_8_1.xml b/gather_program/.idea/libraries/Maven__junit_junit_3_8_1.xml
deleted file mode 100644
index 71b2993d4..000000000
--- a/gather_program/.idea/libraries/Maven__junit_junit_3_8_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__log4j_log4j_1_2_17.xml b/gather_program/.idea/libraries/Maven__log4j_log4j_1_2_17.xml
deleted file mode 100644
index e383c1bf1..000000000
--- a/gather_program/.idea/libraries/Maven__log4j_log4j_1_2_17.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml b/gather_program/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml
deleted file mode 100644
index 17b6e606c..000000000
--- a/gather_program/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml b/gather_program/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml
deleted file mode 100644
index 32bfe3b20..000000000
--- a/gather_program/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml b/gather_program/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml
deleted file mode 100644
index 75070eb48..000000000
--- a/gather_program/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml b/gather_program/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml
deleted file mode 100644
index 0ff136f4a..000000000
--- a/gather_program/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml b/gather_program/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml
deleted file mode 100644
index 1e672600f..000000000
--- a/gather_program/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_7.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_7.xml b/gather_program/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_7.xml
deleted file mode 100644
index 65d0ae68d..000000000
--- a/gather_program/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_7.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_aop_4_1_4_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_aop_4_1_4_RELEASE.xml
deleted file mode 100644
index c1dd843da..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_aop_4_1_4_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_beans_4_1_4_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_beans_4_1_4_RELEASE.xml
deleted file mode 100644
index 09c706d2d..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_beans_4_1_4_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_context_4_1_4_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_context_4_1_4_RELEASE.xml
deleted file mode 100644
index 62d56a595..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_context_4_1_4_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_core_4_1_4_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_core_4_1_4_RELEASE.xml
deleted file mode 100644
index dd905809c..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_core_4_1_4_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_expression_4_1_4_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_expression_4_1_4_RELEASE.xml
deleted file mode 100644
index 5e76b4eb5..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_expression_4_1_4_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml
deleted file mode 100644
index e037dcfa7..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml b/gather_program/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml
deleted file mode 100644
index f7edb5d88..000000000
--- a/gather_program/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/misc.xml b/gather_program/.idea/misc.xml
deleted file mode 100644
index 2e1a5699d..000000000
--- a/gather_program/.idea/misc.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1.8
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/modules.xml b/gather_program/.idea/modules.xml
deleted file mode 100644
index 0c732e1a9..000000000
--- a/gather_program/.idea/modules.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.idea/workspace.xml b/gather_program/.idea/workspace.xml
deleted file mode 100644
index 464ce9784..000000000
--- a/gather_program/.idea/workspace.xml
+++ /dev/null
@@ -1,215 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
- DEFINITION_ORDER
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1492270889131
-
-
- 1492270889131
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/.project b/gather_program/.project
deleted file mode 100644
index 0e41c4683..000000000
--- a/gather_program/.project
+++ /dev/null
@@ -1,29 +0,0 @@
-
-
- gather_program
-
-
-
-
-
- org.eclipse.jdt.core.javabuilder
-
-
-
-
- org.springframework.ide.eclipse.core.springbuilder
-
-
-
-
- org.eclipse.m2e.core.maven2Builder
-
-
-
-
-
- org.springframework.ide.eclipse.core.springnature
- org.eclipse.jdt.core.javanature
- org.eclipse.m2e.core.maven2Nature
-
-
diff --git a/gather_program/bin/.DS_Store b/gather_program/bin/.DS_Store
deleted file mode 100644
index d7714a167..000000000
Binary files a/gather_program/bin/.DS_Store and /dev/null differ
diff --git a/gather_program/bin/gather_projects.sh b/gather_program/bin/gather_projects.sh
deleted file mode 100644
index 9628d7e3f..000000000
--- a/gather_program/bin/gather_projects.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-
-#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
-
-tmp='./target/classes':$tmp
-tmp='./target/gather_program-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-tmp='./bin/resources':$tmp
-CLASSPATH=$tmp:$CLASSPATH
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2 -XX:-UseGCOverheadLimit"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-java $JVM_ARGS -classpath $CLASSPATH org.ossean.gather.process.GatherProcess gather_projects > log/gather_projects.log 2>&1 &
\ No newline at end of file
diff --git a/gather_program/bin/job_requirements.sh b/gather_program/bin/job_requirements.sh
deleted file mode 100644
index 7e1278366..000000000
--- a/gather_program/bin/job_requirements.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-
-#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
-
-tmp='./target/classes':$tmp
-tmp='./target/gather_program-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-tmp='./bin/resources':$tmp
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2 -XX:-UseGCOverheadLimit"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-java $JVM_ARGS -classpath $CLASSPATH org.ossean.gather.process.GatherProcess job_requirements > log/job_requirements.log 2>&1 &
\ No newline at end of file
diff --git a/gather_program/bin/relative_memos.sh b/gather_program/bin/relative_memos.sh
deleted file mode 100644
index 90fa7d9d9..000000000
--- a/gather_program/bin/relative_memos.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-find ./target/classes -name "*.properties"|xargs rm -f
-find ./target/classes -name "*.xml"|xargs rm -f
-find ./target/classes -name "*.dic"|xargs rm -f
-
-#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
-
-tmp='./target/classes':$tmp
-tmp='./target/gather_program-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
-tmp='./bin/resources':$tmp
-CLASSPATH=$tmp:$CLASSPATH
-
-
-echo $CLASSPATH
-JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2 -XX:-UseGCOverheadLimit"
-#echo JVM_ARGS=$JVM_ARGS
-#ulimit -n 400000
-#echo "" > nohup.out
-java $JVM_ARGS -classpath $CLASSPATH org.ossean.gather.process.GatherProcess relative_memos > log/relative_memos.log 2>&1 &
diff --git a/gather_program/bin/resources/applicationContext-myBatis.xml b/gather_program/bin/resources/applicationContext-myBatis.xml
deleted file mode 100644
index a1aa5cab9..000000000
--- a/gather_program/bin/resources/applicationContext-myBatis.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/bin/resources/applicationContext.xml b/gather_program/bin/resources/applicationContext.xml
deleted file mode 100644
index 40b3ec4dc..000000000
--- a/gather_program/bin/resources/applicationContext.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/bin/resources/gather_projects.xml b/gather_program/bin/resources/gather_projects.xml
deleted file mode 100644
index 6a675f038..000000000
--- a/gather_program/bin/resources/gather_projects.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
- TableFlow
- pointers
- github,oschina_project,openhub_project,sourceforge_project,apache,freecode_project
- gather_projects
- id,name,tags,url,url_md5,description,language,source,license,homepage,now(),extracted_time,created_time
-
- id,name,tags,url,url_md5,description,language,source,license,homepage,updated_time,extracted_time,created_time
- 3600000
-
- 1
- 500000
- 500
-
\ No newline at end of file
diff --git a/gather_program/bin/resources/job_requirements.xml b/gather_program/bin/resources/job_requirements.xml
deleted file mode 100644
index 909d343d6..000000000
--- a/gather_program/bin/resources/job_requirements.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
- TableFlow
- pointers
- job_requirements
- job_requirements
- id,title,content,created_time,type,tags,url,url_md5,author,author_url,view_num,review_num,extracted_time,source,similar_position,work_place,experience,salary,scale,domain,finance,style,education
-
- id,title,content,created_time,type,tags,url,url_md5,author,author_url,view_num,review_num,extracted_time,source,similar_position,work_place,experience,salary,scale,domain,finance,style,education
- 3600000
-
- 1
- 500000
- 500
-
\ No newline at end of file
diff --git a/gather_program/bin/resources/log4j.xml b/gather_program/bin/resources/log4j.xml
deleted file mode 100644
index cad5b8ff1..000000000
--- a/gather_program/bin/resources/log4j.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/bin/resources/relative_memos.xml b/gather_program/bin/resources/relative_memos.xml
deleted file mode 100644
index ea2f60ba3..000000000
--- a/gather_program/bin/resources/relative_memos.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
- TableFlow
- pointers
- oschina_question,iteye_blog,51cto_blog,csdn_question,cnblog_news,cnblog_question,csdn_bbs,csdn_blogs,dewen_question,stackoverflow
- relative_memos
- id,title,content,created_time,now(),type,tags,source,url,url_md5,author,author_url,view_num,review_num,extracted_time
- id,title,content,created_time,updated_time,memo_type,tags,source,url,url_md5,author,author_url,view_num,review_num,extracted_time
- 10000
-
-
-
- 1
-
- 500000
-
- 500
-
\ No newline at end of file
diff --git a/gather_program/bin/start_all_flow.sh b/gather_program/bin/start_all_flow.sh
deleted file mode 100644
index 92ebdc3be..000000000
--- a/gather_program/bin/start_all_flow.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-sh bin/gather_projects.sh
-sh bin/relative_memos.sh
-sh bin/job_requirements.sh
-
diff --git a/gather_program/gather_program.iml b/gather_program/gather_program.iml
deleted file mode 100644
index 9f4f5b04e..000000000
--- a/gather_program/gather_program.iml
+++ /dev/null
@@ -1,34 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/gather_program/pom.xml b/gather_program/pom.xml
deleted file mode 100644
index 304f8f1fb..000000000
--- a/gather_program/pom.xml
+++ /dev/null
@@ -1,92 +0,0 @@
-
- 4.0.0
-
- cn.edu.zhanyun
- gather_program
- 0.0.1-SNAPSHOT
-
- gather_program
- http://maven.apache.org
-
-
- ${basedir}/src/main/java
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
- 1.7
- 1.7
-
-
-
- maven-assembly-plugin
- 2.5.1
-
-
- src/main/assembly/assembly.xml
-
-
-
-
-
-
-
- UTF-8
-
-
-
-
- junit
- junit
- 3.8.1
- test
-
-
- org.slf4j
- slf4j-log4j12
- 1.7.7
-
-
- commons-collections
- commons-collections
- 3.2.1
-
-
- org.apache.commons
- commons-io
- 1.3.2
-
-
- org.springframework
- spring-context
- 4.1.4.RELEASE
-
-
- org.apache.commons
- commons-lang3
- 3.1
-
-
- mysql
- mysql-connector-java
- 5.1.18
-
-
- commons-dbcp
- commons-dbcp
- 1.3
-
-
- org.mybatis
- mybatis
- 3.1.1
-
-
- org.mybatis
- mybatis-spring
- 1.1.1
-
-
-
diff --git a/gather_program/sql/edd_pointers.sql b/gather_program/sql/edd_pointers.sql
deleted file mode 100644
index f6e3c33d6..000000000
--- a/gather_program/sql/edd_pointers.sql
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : crawler
-Source Server Version : 50619
-Source Host : 192.168.80.104:3306
-Source Database : extract_result
-
-Target Server Type : MYSQL
-Target Server Version : 50619
-File Encoding : 65001
-
-Date: 2016-02-23 17:31:10
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for edd_pointers
--- ----------------------------
-DROP TABLE IF EXISTS `edd_pointers`;
-CREATE TABLE `edd_pointers` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `SourceTableName` varchar(255) NOT NULL,
- `TargetTableName` varchar(255) NOT NULL,
- `Pointer` int(11) NOT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=23 DEFAULT CHARSET=utf8;
diff --git a/gather_program/sql/gather_projects.sql b/gather_program/sql/gather_projects.sql
deleted file mode 100644
index 382c3bfd9..000000000
--- a/gather_program/sql/gather_projects.sql
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : localhost:3306
-Source Database : ossean_new
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-02-19 14:07:15
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for gather_projects
--- ----------------------------
-DROP TABLE IF EXISTS `gather_projects`;
-CREATE TABLE `gather_projects` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- `tags` varchar(2000) DEFAULT '',
- `url` text,
- `url_md5` varchar(255) DEFAULT NULL,
- `description` text,
- `language` varchar(255) DEFAULT NULL,
- `source` varchar(255) DEFAULT NULL,
- `tags_for_search` varchar(2000) DEFAULT NULL,
- `synonyms` varchar(255) DEFAULT NULL,
- `license` text,
- `view_num_ossean` int(11) DEFAULT '0',
- `homepage` varchar(255) DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `extracted_time` datetime DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `composite_score` double DEFAULT '0',
- `relative_memos_num` int(11) DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `filtration` int(11) DEFAULT NULL,
- `update_mark` int(11) DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `index_open_source_projects_on_ossean_score` (`composite_score`) USING BTREE,
- KEY `index_open_source_projects_on_name` (`name`) USING BTREE,
- KEY `index_open_source_projects_on_relative_memos_num` (`relative_memos_num`) USING BTREE,
- KEY `index_open_source_projects_on_composite_score` (`composite_score`) USING BTREE,
- KEY `index_open_source_projects_on_source` (`source`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/gather_program/sql/job_requirements.sql b/gather_program/sql/job_requirements.sql
deleted file mode 100644
index cc98f1037..000000000
--- a/gather_program/sql/job_requirements.sql
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : crawler
-Source Server Version : 50619
-Source Host : 192.168.80.104:3306
-Source Database : extracted_data
-
-Target Server Type : MYSQL
-Target Server Version : 50619
-File Encoding : 65001
-
-Date: 2016-01-26 15:16:20
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for job_requirements
--- ----------------------------
-DROP TABLE IF EXISTS `job_requirements`;
-CREATE TABLE `job_requirements` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `title` varchar(255) NOT NULL,
- `content` longtext,
- `created_time` datetime DEFAULT NULL,
- `type` varchar(100) DEFAULT NULL,
- `tags` varchar(255) DEFAULT NULL,
- `url` varchar(255) NOT NULL,
- `url_md5` varchar(34) DEFAULT NULL,
- `author` varchar(100) DEFAULT NULL,
- `author_url` varchar(150) DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `view_num` int(11) unsigned DEFAULT NULL,
- `review_num` int(11) unsigned DEFAULT NULL,
- `extracted_time` datetime NOT NULL,
- `source` varchar(255) DEFAULT NULL,
- `similar_position` varchar(255) DEFAULT NULL,
- `work_place` varchar(255) DEFAULT NULL,
- `experience` varchar(255) DEFAULT NULL,
- `salary` varchar(255) DEFAULT NULL,
- `scale` varchar(255) DEFAULT NULL,
- `domain` varchar(255) DEFAULT NULL,
- `finance` varchar(255) DEFAULT NULL,
- `style` varchar(255) DEFAULT NULL,
- `education` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/gather_program/sql/open_source_projects.sql b/gather_program/sql/open_source_projects.sql
deleted file mode 100644
index a9f68e5e0..000000000
--- a/gather_program/sql/open_source_projects.sql
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : 127.0.0.1:3306
-Source Database : ossean_new
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-11-17 18:44:25
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for open_source_projects
--- ----------------------------
-DROP TABLE IF EXISTS `open_source_projects`;
-CREATE TABLE `open_source_projects` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- `tags` varchar(2000) DEFAULT '',
- `url` text,
- `url_md5` varchar(255) DEFAULT NULL,
- `description` text,
- `language` varchar(255) DEFAULT NULL,
- `source` varchar(255) DEFAULT NULL,
- `tags_for_search` varchar(2000) DEFAULT NULL,
- `synonyms` varchar(255) DEFAULT NULL,
- `license` text,
- `view_num_ossean` int(11) DEFAULT '0',
- `homepage` varchar(255) DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `extracted_time` datetime DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `composite_score` double DEFAULT '0',
- `relative_memos_num` int(11) DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `filtration` int(11) DEFAULT NULL,
- `update_mark` int(11) DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `name_index` (`name`) USING BTREE,
- KEY `relative_memos_num_index` (`relative_memos_num`) USING BTREE,
- KEY `composite_score_index` (`composite_score`) USING BTREE,
- KEY `source_index` (`source`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=210 DEFAULT CHARSET=utf8;
diff --git a/gather_program/sql/ossean_production.sql b/gather_program/sql/ossean_production.sql
deleted file mode 100644
index adb138f73..000000000
--- a/gather_program/sql/ossean_production.sql
+++ /dev/null
@@ -1,2479 +0,0 @@
--- MySQL dump 10.13 Distrib 5.7.15, for Linux (x86_64)
---
--- Host: localhost Database: ossean_production
--- ------------------------------------------------------
--- Server version 5.7.15-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
-/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `edd_pointers`
---
-
-DROP TABLE IF EXISTS `edd_pointers`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `edd_pointers` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `SourceTableName` varchar(255) NOT NULL,
- `TargetTableName` varchar(255) NOT NULL,
- `Pointer` int(11) NOT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=15 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `edd_relations`
---
-
-DROP TABLE IF EXISTS `edd_relations`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `edd_relations` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `gather_projects_ids` varchar(255) NOT NULL,
- PRIMARY KEY (`id`),
- KEY `gather_projects_ids` (`gather_projects_ids`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1186946 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `gather_projects`
---
-
-DROP TABLE IF EXISTS `gather_projects`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `gather_projects` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- `tags` varchar(2000) DEFAULT NULL,
- `url` text,
- `url_md5` varchar(255) DEFAULT NULL,
- `description` text,
- `language` varchar(1000) DEFAULT NULL,
- `source` varchar(255) DEFAULT NULL,
- `tags_for_search` varchar(2000) DEFAULT NULL,
- `synonyms` varchar(255) DEFAULT NULL,
- `license` text,
- `view_num_ossean` int(11) DEFAULT '0',
- `homepage` varchar(255) DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `extracted_time` datetime DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `composite_score` double DEFAULT '0',
- `relative_memos_num` int(11) DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `filtration` int(11) DEFAULT '0',
- `update_mark` int(11) DEFAULT '0',
- PRIMARY KEY (`id`),
- UNIQUE KEY `unique_urlmd5` (`url_md5`),
- KEY `homepage_index` (`homepage`)
-) ENGINE=InnoDB AUTO_INCREMENT=1135075 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `gather_projects_withsynonyms`
---
-
-DROP TABLE IF EXISTS `gather_projects_withsynonyms`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `gather_projects_withsynonyms` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- `tags` varchar(2000) DEFAULT '',
- `url` text,
- `url_md5` varchar(255) DEFAULT NULL,
- `description` text,
- `language` varchar(255) DEFAULT NULL,
- `source` varchar(255) DEFAULT NULL,
- `tags_for_search` varchar(2000) DEFAULT NULL,
- `synonyms` varchar(255) DEFAULT NULL,
- `license` text,
- `view_num_ossean` int(11) DEFAULT '0',
- `homepage` varchar(255) DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `extracted_time` datetime DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `composite_score` double DEFAULT '0',
- `relative_memos_num` int(11) DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `filtration` int(11) DEFAULT NULL,
- `update_mark` int(11) DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `index_open_source_projects_on_ossean_score` (`composite_score`) USING BTREE,
- KEY `index_open_source_projects_on_name` (`name`) USING BTREE,
- KEY `index_open_source_projects_on_relative_memos_num` (`relative_memos_num`) USING BTREE,
- KEY `index_open_source_projects_on_composite_score` (`composite_score`) USING BTREE,
- KEY `index_open_source_projects_on_source` (`source`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1135052 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `highcharts`
---
-
-DROP TABLE IF EXISTS `highcharts`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `highcharts` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `match_time` datetime DEFAULT NULL,
- `min_salary` int(11) DEFAULT NULL,
- `max_salary` int(11) DEFAULT NULL,
- `experience` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
- `city` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `record_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `distinct_experience` (`osp_id`,`experience`) USING BTREE,
- KEY `cal_min_by_experience` (`osp_id`,`experience`,`min_salary`) USING BTREE,
- KEY `cal_max_by_experience` (`osp_id`,`experience`,`max_salary`) USING BTREE,
- KEY `distinct_city` (`city`,`osp_id`) USING BTREE,
- KEY `cal_min_by_city` (`osp_id`,`city`,`min_salary`) USING BTREE,
- KEY `cal_max_by_city` (`osp_id`,`city`,`max_salary`) USING BTREE,
- KEY `find_experience_by_city` (`city`,`experience`) USING BTREE,
- KEY `cal_min_by_city_experience` (`experience`,`city`,`min_salary`) USING BTREE,
- KEY `cal_max_by_city_experience` (`experience`,`city`,`max_salary`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `hot_words`
---
-
-DROP TABLE IF EXISTS `hot_words`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `hot_words` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) DEFAULT '0',
- `name` varchar(255) DEFAULT '',
- `weight` float DEFAULT '0',
- `created_at` datetime NOT NULL,
- `updated_at` datetime NOT NULL,
- PRIMARY KEY (`id`),
- KEY `index_hot_words_on_osp_id` (`osp_id`) USING BTREE,
- KEY `osp_id_and_hotword_name` (`osp_id`,`name`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=174121 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `job_requirements`
---
-
-DROP TABLE IF EXISTS `job_requirements`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `job_requirements` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `title` varchar(255) NOT NULL,
- `content` longtext,
- `created_time` datetime DEFAULT NULL,
- `type` varchar(100) DEFAULT NULL,
- `tags` varchar(255) DEFAULT NULL,
- `url` varchar(255) NOT NULL,
- `url_md5` varchar(34) DEFAULT NULL,
- `author` varchar(100) DEFAULT NULL,
- `author_url` varchar(150) DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `view_num` int(11) unsigned DEFAULT NULL,
- `review_num` int(11) unsigned DEFAULT NULL,
- `extracted_time` datetime NOT NULL,
- `source` varchar(255) DEFAULT NULL,
- `similar_position` varchar(2000) DEFAULT NULL,
- `work_place` varchar(255) DEFAULT NULL,
- `experience` varchar(255) DEFAULT NULL,
- `salary` varchar(255) DEFAULT NULL,
- `scale` varchar(255) DEFAULT NULL,
- `domain` varchar(255) DEFAULT NULL,
- `finance` varchar(255) DEFAULT NULL,
- `style` varchar(255) DEFAULT NULL,
- `education` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=6389147 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `memo_taggings`
---
-
-DROP TABLE IF EXISTS `memo_taggings`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `memo_taggings` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `tag_id` int(11) NOT NULL,
- `taggable_id` int(11) NOT NULL,
- `taggable_type` varchar(255) NOT NULL,
- `tagger_id` int(11) DEFAULT NULL,
- `tagger_type` varchar(255) DEFAULT NULL,
- `context` varchar(128) DEFAULT NULL,
- `created_at` datetime DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `disagree_num` int(11) DEFAULT '0',
- `tag_source` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `index_taggings_on_tag_id_and_taggable_id_and_taggable_type` (`tag_id`,`taggable_id`,`taggable_type`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=43045307 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `new_osp_zyr`
---
-
-DROP TABLE IF EXISTS `new_osp_zyr`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `new_osp_zyr` (
- `id` int(11) NOT NULL DEFAULT '0',
- `name` varchar(255) CHARACTER SET utf8 NOT NULL,
- `tags` varchar(2000) CHARACTER SET utf8 DEFAULT '',
- `url` text CHARACTER SET utf8,
- `url_md5` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
- `description` text CHARACTER SET utf8,
- `language` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
- `source` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
- `tags_for_search` varchar(2000) CHARACTER SET utf8 DEFAULT NULL,
- `synonyms` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
- `license` text CHARACTER SET utf8,
- `view_num_ossean` int(11) DEFAULT '0',
- `homepage` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `extracted_time` datetime DEFAULT NULL,
- `category` varchar(255) CHARACTER SET utf8 DEFAULT NULL,
- `composite_score` double DEFAULT '0',
- `relative_memos_num` int(11) DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `filtration` int(11) DEFAULT NULL,
- `update_mark` int(11) DEFAULT NULL
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `open_source_project_popularities`
---
-
-DROP TABLE IF EXISTS `open_source_project_popularities`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `open_source_project_popularities` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) DEFAULT NULL,
- `year_col` int(11) DEFAULT '0',
- `month_col` int(11) DEFAULT '0',
- `popularity` int(11) DEFAULT '0',
- `created_at` datetime NOT NULL,
- `updated_at` datetime NOT NULL,
- PRIMARY KEY (`id`),
- KEY `osp_pop_index` (`osp_id`,`year_col`,`month_col`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `open_source_projects`
---
-
-DROP TABLE IF EXISTS `open_source_projects`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `open_source_projects` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- `tags` varchar(2000) DEFAULT '',
- `url` text,
- `url_md5` varchar(255) DEFAULT NULL,
- `description` text,
- `language` varchar(255) DEFAULT NULL,
- `source` varchar(255) DEFAULT NULL,
- `tags_for_search` varchar(2000) DEFAULT NULL,
- `synonyms` varchar(255) DEFAULT NULL,
- `license` text,
- `view_num_ossean` int(11) DEFAULT '0',
- `homepage` varchar(255) DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `extracted_time` datetime DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `composite_score` double DEFAULT '0',
- `relative_memos_num` int(11) DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `filtration` int(11) DEFAULT NULL,
- `update_mark` int(11) DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `name_index` (`name`) USING BTREE,
- KEY `relative_memos_num_index` (`relative_memos_num`) USING BTREE,
- KEY `composite_score_index` (`composite_score`) USING BTREE,
- KEY `source_index` (`source`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1135075 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `osp_with_synonyms`
---
-
-DROP TABLE IF EXISTS `osp_with_synonyms`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `osp_with_synonyms` (
- `id` int(11) NOT NULL DEFAULT '0',
- `name` varchar(255) CHARACTER SET utf8 NOT NULL,
- `url` text CHARACTER SET utf8,
- `synonyms` varchar(255) CHARACTER SET utf8 DEFAULT NULL
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `ossean_monitors`
---
-
-DROP TABLE IF EXISTS `ossean_monitors`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `ossean_monitors` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `website` varchar(255) DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `crawler_charge` varchar(255) DEFAULT NULL,
- `day_crawler` decimal(10,0) DEFAULT NULL,
- `week_crawler` decimal(10,0) DEFAULT NULL,
- `month_crawler` decimal(10,0) DEFAULT NULL,
- `crawler_sum` decimal(10,0) DEFAULT NULL,
- `extractor_charge` varchar(255) DEFAULT NULL,
- `day_extractor` decimal(10,0) DEFAULT NULL,
- `week_extractor` decimal(10,0) DEFAULT NULL,
- `month_extractor` decimal(10,0) DEFAULT NULL,
- `extract_rate` varchar(50) DEFAULT NULL,
- `flow_num` varchar(255) DEFAULT NULL,
- `flow_rate` decimal(10,3) DEFAULT '0.000',
- `match_num` decimal(10,0) DEFAULT NULL,
- `new_memos` decimal(10,0) DEFAULT NULL,
- `week_new_memos` decimal(10,0) DEFAULT NULL,
- `month_new_memos` decimal(10,0) DEFAULT NULL,
- `total_num` decimal(10,0) DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `pointers`
---
-
-DROP TABLE IF EXISTS `pointers`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `pointers` (
- `Id` int(11) unsigned NOT NULL AUTO_INCREMENT,
- `SourceTableName` varchar(255) DEFAULT NULL,
- `TargetTableName` varchar(255) DEFAULT NULL,
- `Pointer` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`Id`)
-) ENGINE=InnoDB AUTO_INCREMENT=26 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relation_recommends`
---
-
-DROP TABLE IF EXISTS `relation_recommends`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relation_recommends` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) DEFAULT NULL,
- `relative_osp_id` int(11) DEFAULT NULL,
- `weight` float DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `replies_num` int(11) DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `vote_up_num` int(11) DEFAULT NULL,
- `collection_num` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `category` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `composite_id_by_ospId_relativeMemoId` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `match_weight_sort` (`match_weight`) USING BTREE,
- KEY `find_by_osp_id` (`osp_id`) USING BTREE,
- KEY `osp_id_weight_query` (`osp_id`,`match_weight`) USING BTREE,
- KEY `relative_memo_to_osp_osp_id_created` (`osp_id`,`created_time`) USING BTREE,
- KEY `relative_memo_to_osp_osp_id_replies_num` (`osp_id`,`replies_num`) USING BTREE,
- KEY `check_if_synchronized` (`has_synchronized`) USING BTREE,
- KEY `index_relative_memo_to_open_source_projects_on_relative_memo_id` (`relative_memo_id`) USING BTREE,
- KEY `relative_memo_to_osp_osp_id_view_num_crawled` (`osp_id`,`view_num_crawled`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_1`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_1`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_1` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_1` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=3974409 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_10`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_10`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_10` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_10` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=730314 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_11`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_11`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_11` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_11` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1304234 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_12`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_12`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_12` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_12` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=556061 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_13`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_13`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_13` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_13` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1203208 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_14`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_14`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_14` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_14` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=612656 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_15`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_15`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_15` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_15` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=671774 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_16`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_16`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_16` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_16` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=734878 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_17`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_17`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_17` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_17` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1560378 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_18`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_18`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_18` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_18` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=694268 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_19`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_19`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_19` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_19` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=353185 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_2`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_2`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_2` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_2` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2389705 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_20`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_20`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_20` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_20` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_20` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=962325 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_21`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_21`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_21` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_21` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_21` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1839393 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_22`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_22`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_22` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_22` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_22` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1105400 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_23`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_23`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_23` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_23` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_23` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=297837 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_24`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_24`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_24` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_24` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_24` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1339421 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_25`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_25`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_25` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_25` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_25` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1383408 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_26`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_26`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_26` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_26` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_26` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=863884 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_27`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_27`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_27` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_27` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_27` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1085797 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_28`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_28`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_28` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_28` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_28` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1529334 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_29`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_29`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_29` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_29` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_29` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1190174 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_3`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_3`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_3` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_3` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2034974 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_30`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_30`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_30` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_30` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_30` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=736820 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_31`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_31`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_31` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_31` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_31` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=892039 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_32`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_32`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_32` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_32` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_32` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=482672 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_33`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_33`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_33` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_33` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_33` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1495075 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_34`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_34`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_34` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_34` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_34` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1274887 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_35`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_35`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_35` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_35` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_35` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1942083 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_36`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_36`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_36` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_36` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_36` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=804690 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_37`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_37`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_37` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_37` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_37` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=954829 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_38`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_38`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_38` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_38` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_38` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1783198 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_39`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_39`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_39` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_39` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_39` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2081264 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_4`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_4`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_4` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_4` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1432651 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_40`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_40`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_40` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_40` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_40` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1472652 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_41`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_41`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_41` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_41` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_41` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1831893 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_42`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_42`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_42` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_42` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_42` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1633917 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_43`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_43`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_43` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_43` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_43` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1516313 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_44`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_44`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_44` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_44` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_44` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1284050 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_45`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_45`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_45` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_45` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_45` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1207280 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_46`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_46`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_46` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_46` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_46` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2006772 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_47`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_47`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_47` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_47` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_47` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1457973 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_48`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_48`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_48` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_48` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_48` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1484049 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_49`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_49`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_49` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_49` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_49` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=859824 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_5`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_5`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_5` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_5` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2267223 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_50`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_50`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_50` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_50` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_50` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1475363 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_51`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_51`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_51` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_51` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_51` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=986020 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_52`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_52`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_52` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_52` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_52` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1549959 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_53`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_53`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_53` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_53` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_53` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1693623 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_54`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_54`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_54` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_54` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_54` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2801968 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_55`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_55`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_55` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_55` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_55` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1277825 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_56`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_56`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_56` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_56` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_56` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1001984 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_57`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_57`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_57` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_57` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_57` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=828128 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_58`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_58`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_58` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_58` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_58` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=958565 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_59`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_59`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_59` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_59` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_59` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2064180 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_6`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_6`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_6` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_6` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1378226 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_60`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_60`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_60` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_60` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_60` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1126582 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_61`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_61`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_61` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_61` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_61` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=579753 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_62`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_62`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_62` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_62` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_62` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=183959 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_63`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_63`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_63` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_63` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_63` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=270407 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_64`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_64`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_64` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_64` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_64` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=820077 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_65`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_65`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_65` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_65` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_65` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=396998 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_66`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_66`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_66` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_66` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_66` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1222434 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_67`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_67`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_67` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_67` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_67` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=247580 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_68`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_68`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_68` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_68` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_68` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=889515 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_69`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_69`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_69` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_69` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_69` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=751688 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_7`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_7`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_7` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_7` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1617647 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_70`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_70`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_70` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index_70` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_70` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=55528090 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_8`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_8`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_8` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_8` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2268582 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memo_to_open_source_projects_9`
---
-
-DROP TABLE IF EXISTS `relative_memo_to_open_source_projects_9`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memo_to_open_source_projects_9` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_memo_id` int(11) NOT NULL,
- `match_weight` float DEFAULT '0',
- `replies_num` int(11) DEFAULT NULL,
- `view_num_crawled` int(11) DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `view_num_trustie` int(11) DEFAULT NULL,
- `has_synchronized` tinyint(1) DEFAULT '0',
- `created_time` datetime DEFAULT NULL,
- `match_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `osp_id_relative_memo_id_unique_index` (`osp_id`,`relative_memo_id`) USING BTREE,
- KEY `relative_memo_id_index` (`relative_memo_id`) USING BTREE,
- KEY `has_synchronized_index_9` (`has_synchronized`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1668808 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `relative_memos`
---
-
-DROP TABLE IF EXISTS `relative_memos`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `relative_memos` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `title` varchar(255) NOT NULL DEFAULT '',
- `content` longtext,
- `created_time` datetime DEFAULT NULL,
- `updated_time` datetime DEFAULT NULL,
- `memo_type` varchar(255) DEFAULT NULL,
- `tags` varchar(2000) DEFAULT '',
- `source` varchar(255) DEFAULT '',
- `url` varchar(255) DEFAULT '',
- `url_md5` varchar(36) DEFAULT NULL,
- `author` varchar(255) DEFAULT NULL,
- `author_url` varchar(255) DEFAULT NULL,
- `view_num` int(11) DEFAULT '0',
- `review_num` int(11) DEFAULT '0',
- `view_num_ossean` int(11) DEFAULT '0',
- `extracted_time` datetime NOT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `index_relative_memos_on_url_md5` (`url_md5`) USING BTREE,
- UNIQUE KEY `unique_urlmd5` (`url_md5`),
- KEY `search_memo_type` (`memo_type`) USING BTREE,
- KEY `index_relative_memos_on_url` (`url`) USING BTREE,
- KEY `index_relative_memos_on_source` (`source`) USING BTREE,
- KEY `created_time_index` (`created_time`),
- KEY `type_created_time_index` (`memo_type`,`created_time`)
-) ENGINE=InnoDB AUTO_INCREMENT=6719061 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `schema_migrations`
---
-
-DROP TABLE IF EXISTS `schema_migrations`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `schema_migrations` (
- `version` varchar(255) NOT NULL,
- UNIQUE KEY `unique_schema_migrations` (`version`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `settings`
---
-
-DROP TABLE IF EXISTS `settings`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `settings` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL DEFAULT '',
- `value` text,
- `updated_on` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `index_settings_on_name` (`name`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `simility_recommend_up_downs`
---
-
-DROP TABLE IF EXISTS `simility_recommend_up_downs`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `simility_recommend_up_downs` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) NOT NULL,
- `relative_osp_id` int(11) NOT NULL,
- `up_down_value` int(11) DEFAULT '0',
- `last_up_time` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `osp_id` (`osp_id`,`up_down_value`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `simility_recommends`
---
-
-DROP TABLE IF EXISTS `simility_recommends`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `simility_recommends` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `osp_id` int(11) DEFAULT NULL,
- `relative_osp_id` int(11) DEFAULT NULL,
- `weight` float DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `solr_incremental_index_cursors`
---
-
-DROP TABLE IF EXISTS `solr_incremental_index_cursors`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `solr_incremental_index_cursors` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- `cursor` int(11) DEFAULT '0',
- `created_at` datetime NOT NULL,
- `updated_at` datetime NOT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `synonymmings`
---
-
-DROP TABLE IF EXISTS `synonymmings`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `synonymmings` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `synonym_id` int(11) DEFAULT NULL,
- `synonymming_id` int(11) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `synonym_id` (`synonym_id`,`synonymming_id`) USING BTREE,
- KEY `synonymming_id` (`synonymming_id`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=2060988 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `synonyms`
---
-
-DROP TABLE IF EXISTS `synonyms`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `synonyms` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `name` (`name`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=1290537 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `taggings`
---
-
-DROP TABLE IF EXISTS `taggings`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `taggings` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `tag_id` int(11) NOT NULL,
- `taggable_id` int(11) NOT NULL,
- `taggable_type` varchar(255) NOT NULL,
- `tagger_id` int(11) DEFAULT NULL,
- `tagger_type` varchar(255) DEFAULT NULL,
- `context` varchar(128) DEFAULT NULL,
- `created_at` datetime DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `disagree_num` int(11) DEFAULT '0',
- `tag_source` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `index_taggings_on_tag_id_and_taggable_id_and_taggable_type` (`tag_id`,`taggable_id`,`taggable_type`) USING BTREE,
- KEY `taggab_id_type_context_index` (`taggable_id`,`taggable_type`,`context`)
-) ENGINE=InnoDB AUTO_INCREMENT=1196462 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `tags`
---
-
-DROP TABLE IF EXISTS `tags`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `tags` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `unique_name_index` (`name`) USING BTREE,
- KEY `index_tags_on_name` (`name`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=4588934 DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-
---
--- Table structure for table `versions`
---
-
-DROP TABLE IF EXISTS `versions`;
-/*!40101 SET @saved_cs_client = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `versions` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `project_id` int(11) NOT NULL DEFAULT '0',
- `name` varchar(255) NOT NULL DEFAULT '',
- `description` varchar(255) DEFAULT '',
- `effective_date` date DEFAULT NULL,
- `created_on` datetime DEFAULT NULL,
- `updated_on` datetime DEFAULT NULL,
- `wiki_page_title` varchar(255) DEFAULT NULL,
- `status` varchar(255) DEFAULT 'open',
- `sharing` varchar(255) NOT NULL DEFAULT 'none',
- PRIMARY KEY (`id`),
- KEY `versions_project_id` (`project_id`) USING BTREE,
- KEY `index_versions_on_sharing` (`sharing`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
-/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2017-01-03 14:57:01
diff --git a/gather_program/sql/pk_control_posts.sql b/gather_program/sql/pk_control_posts.sql
deleted file mode 100644
index 5b0717a26..000000000
--- a/gather_program/sql/pk_control_posts.sql
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : localhost:3306
-Source Database : ossean_new
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-01-25 19:54:21
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for pk_control_posts
--- ----------------------------
-DROP TABLE IF EXISTS `pk_control_posts`;
-CREATE TABLE `pk_control_posts` (
- `id` int(11) NOT NULL,
- `url_md5` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
diff --git a/gather_program/sql/pk_control_projects.sql b/gather_program/sql/pk_control_projects.sql
deleted file mode 100644
index b1c6e1a60..000000000
--- a/gather_program/sql/pk_control_projects.sql
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : localhost:3306
-Source Database : ossean_new
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-01-25 19:54:29
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for pk_control_projects
--- ----------------------------
-DROP TABLE IF EXISTS `pk_control_projects`;
-CREATE TABLE `pk_control_projects` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `url_md5` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1;
diff --git a/gather_program/sql/pointers.sql b/gather_program/sql/pointers.sql
deleted file mode 100644
index bd769cc0b..000000000
--- a/gather_program/sql/pointers.sql
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : localhost:3306
-Source Database : ossean_new
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-01-25 19:54:39
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for pointers
--- ----------------------------
-DROP TABLE IF EXISTS `pointers`;
-CREATE TABLE `pointers` (
- `Id` int(11) unsigned NOT NULL AUTO_INCREMENT,
- `SourceTableName` varchar(255) DEFAULT NULL,
- `TargetTableName` varchar(255) DEFAULT NULL,
- `Pointer` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`Id`)
-) ENGINE=InnoDB AUTO_INCREMENT=52 DEFAULT CHARSET=utf8;
diff --git a/gather_program/sql/settings.sql b/gather_program/sql/settings.sql
deleted file mode 100644
index 369b5f171..000000000
--- a/gather_program/sql/settings.sql
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : 127.0.0.1:3306
-Source Database : ossean_new
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-11-15 20:04:15
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for settings
--- ----------------------------
-DROP TABLE IF EXISTS `settings`;
-CREATE TABLE `settings` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `name` varchar(255) NOT NULL DEFAULT '',
- `value` text,
- `updated_on` datetime DEFAULT NULL,
- PRIMARY KEY (`id`),
- KEY `index_settings_on_name` (`name`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/gather_program/sql/taggings.sql b/gather_program/sql/taggings.sql
deleted file mode 100644
index 543b810c1..000000000
--- a/gather_program/sql/taggings.sql
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
-Navicat MySQL Data Transfer
-
-Source Server : ossean
-Source Server Version : 50535
-Source Host : 127.0.0.1:3306
-Source Database : ossean_production
-
-Target Server Type : MYSQL
-Target Server Version : 50535
-File Encoding : 65001
-
-Date: 2016-11-13 22:21:05
-*/
-
-SET FOREIGN_KEY_CHECKS=0;
-
--- ----------------------------
--- Table structure for taggings
--- ----------------------------
-DROP TABLE IF EXISTS `taggings`;
-CREATE TABLE `taggings` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `tag_id` int(11) NOT NULL,
- `taggable_id` int(11) NOT NULL,
- `taggable_type` varchar(255) NOT NULL,
- `tagger_id` int(11) DEFAULT NULL,
- `tagger_type` varchar(255) DEFAULT NULL,
- `context` varchar(128) DEFAULT NULL,
- `created_at` datetime DEFAULT NULL,
- `created_time` datetime DEFAULT NULL,
- `disagree_num` int(11) DEFAULT '0',
- `tag_source` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `index_taggings_on_tag_id_and_taggable_id_and_taggable_type` (`tag_id`,`taggable_id`,`taggable_type`) USING BTREE
-) ENGINE=InnoDB AUTO_INCREMENT=19315557 DEFAULT CHARSET=utf8;
diff --git a/gather_program/src/.DS_Store b/gather_program/src/.DS_Store
deleted file mode 100644
index 117144022..000000000
Binary files a/gather_program/src/.DS_Store and /dev/null differ
diff --git a/gather_program/src/main/.DS_Store b/gather_program/src/main/.DS_Store
deleted file mode 100644
index fe10919bc..000000000
Binary files a/gather_program/src/main/.DS_Store and /dev/null differ
diff --git a/gather_program/src/main/assembly/assembly.xml b/gather_program/src/main/assembly/assembly.xml
deleted file mode 100644
index baf8ba912..000000000
--- a/gather_program/src/main/assembly/assembly.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
- jar-with-dependencies-without-resources
-
- dir
-
- false
-
-
- /
- false
- false
- runtime
-
-
- /
- false
- false
- system
-
-
-
\ No newline at end of file
diff --git a/gather_program/src/main/java/.DS_Store b/gather_program/src/main/java/.DS_Store
deleted file mode 100644
index 77aa5a91a..000000000
Binary files a/gather_program/src/main/java/.DS_Store and /dev/null differ
diff --git a/gather_program/src/main/java/org/.DS_Store b/gather_program/src/main/java/org/.DS_Store
deleted file mode 100644
index d1ef1e22e..000000000
Binary files a/gather_program/src/main/java/org/.DS_Store and /dev/null differ
diff --git a/gather_program/src/main/java/org/ossean/.DS_Store b/gather_program/src/main/java/org/ossean/.DS_Store
deleted file mode 100644
index 2b6bf0e22..000000000
Binary files a/gather_program/src/main/java/org/ossean/.DS_Store and /dev/null differ
diff --git a/gather_program/src/main/java/org/ossean/gather/.DS_Store b/gather_program/src/main/java/org/ossean/gather/.DS_Store
deleted file mode 100644
index c1e5b4e03..000000000
Binary files a/gather_program/src/main/java/org/ossean/gather/.DS_Store and /dev/null differ
diff --git a/gather_program/src/main/java/org/ossean/gather/model/Configure.java b/gather_program/src/main/java/org/ossean/gather/model/Configure.java
deleted file mode 100644
index a8c21c314..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/Configure.java
+++ /dev/null
@@ -1,125 +0,0 @@
-package org.ossean.gather.model;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.InvalidPropertiesFormatException;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.log4j.Logger;
-
-public class Configure {
-
- private Properties prop = new Properties();
- Logger log4j = Logger.getLogger(Configure.class);
- private String pointerTableName;
- private List sourceTableNameList = new ArrayList();
- private String targetTableName;
- private String sourceFields;
- private String targetFields;
- private int idsIncrement;
- private int waitDataTime;
- private String andWhere;
-
- //自定义构造函数
- public Configure(String confPath) {
- confPath = confPath + ".xml";
- try {
- URL url = ClassLoader.getSystemResource(confPath);
- InputStream is = url.openStream();
- prop.loadFromXML(is);
- } catch (FileNotFoundException e) {
- log4j.error("Configure FileNotFoundException !");
- e.printStackTrace();
- } catch (InvalidPropertiesFormatException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- config();
- }
-
- private void config() {
- this.pointerTableName=prop.getProperty("pointerTableName");
- String sourceTableNameString = prop.getProperty("sourceTableName");
- String []sourceTableNameArray = sourceTableNameString.split(",");
- for(String sourceTableName : sourceTableNameArray){
- this.sourceTableNameList.add(sourceTableName);
- }
- this.sourceFields=prop.getProperty("sourceFields");
- this.targetTableName=prop.getProperty("targetTableName");
- this.targetFields=prop.getProperty("targetFields");
- this.idsIncrement=Integer.parseInt(prop.getProperty("idsIncrement"));
- this.waitDataTime=Integer.parseInt(prop.getProperty("waitDataTime"));
- this.andWhere=prop.getProperty("andWhere");
- }
-
- public Properties getProp() {
- return prop;
- }
- public void setProp(Properties prop) {
- this.prop = prop;
- }
- public Logger getLog4j() {
- return log4j;
- }
- public void setLog4j(Logger log4j) {
- this.log4j = log4j;
- }
- public String getPointerTableName() {
- return pointerTableName;
- }
- public void setPointerTableName(String pointerTableName) {
- this.pointerTableName = pointerTableName;
- }
-
- public List getSourceTableNameList() {
- return sourceTableNameList;
- }
-
- public void setSourceTableNameList(List sourceTableNameList) {
- this.sourceTableNameList = sourceTableNameList;
- }
-
- public String getTargetTableName() {
- return targetTableName;
- }
- public void setTargetTableName(String targetTableName) {
- this.targetTableName = targetTableName;
- }
- public String getSourceFields() {
- return sourceFields;
- }
- public void setSourceFields(String sourceFields) {
- this.sourceFields = sourceFields;
- }
- public String getTargetFields() {
- return targetFields;
- }
- public void setTargetFields(String targetFields) {
- this.targetFields = targetFields;
- }
- public int getIdsIncrement() {
- return idsIncrement;
- }
- public void setIdsIncrement(int idsIncrement) {
- this.idsIncrement = idsIncrement;
- }
- public int getWaitDataTime() {
- return waitDataTime;
- }
- public void setWaitDataTime(int waitDataTime) {
- this.waitDataTime = waitDataTime;
- }
- public String getAndWhere() {
- return andWhere;
- }
- public void setAndWhere(String andWhere) {
- this.andWhere = andWhere;
- }
-
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/GatherProject.java b/gather_program/src/main/java/org/ossean/gather/model/GatherProject.java
deleted file mode 100644
index 9b6ff0040..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/GatherProject.java
+++ /dev/null
@@ -1,140 +0,0 @@
-package org.ossean.gather.model;
-
-public class GatherProject {
-
- private int id;
- private String name;
- private String tags;
- private String url;
- private String url_md5; //添加urlMD5作为主键
- private String description;
- private String language;
- private String source;
- private String tags_for_search;
- private String synonyms;
- private String license;
- private int view_num_ossean;
- private String homepage;
- private String updated_time;
- private String extracted_time;
- private int composite_score;
- private int relative_memos_num;
- private String created_time;
- private int update_mark;
-
- public int getUpdate_mark() {
- return update_mark;
- }
- public void setUpdate_mark(int update_mark) {
- this.update_mark = update_mark;
- }
- public int getId() {
- return id;
- }
- public void setId(int id) {
- this.id = id;
- }
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
- public String getTags() {
- return tags;
- }
- public void setTags(String tags) {
- this.tags = tags;
- }
- public String getUrl() {
- return url;
- }
- public void setUrl(String url) {
- this.url = url;
- }
- public String getUrl_md5() {
- return url_md5;
- }
- public void setUrl_md5(String url_md5) {
- this.url_md5 = url_md5;
- }
- public String getDescription() {
- return description;
- }
- public void setDescription(String description) {
- this.description = description;
- }
- public String getLanguage() {
- return language;
- }
- public void setLanguage(String language) {
- this.language = language;
- }
- public String getSource() {
- return source;
- }
- public void setSource(String source) {
- this.source = source;
- }
- public String getTags_for_search() {
- return tags_for_search;
- }
- public void setTags_for_search(String tags_for_search) {
- this.tags_for_search = tags_for_search;
- }
- public String getSynonyms() {
- return synonyms;
- }
- public void setSynonyms(String synonyms) {
- this.synonyms = synonyms;
- }
- public String getLicense() {
- return license;
- }
- public void setLicense(String license) {
- this.license = license;
- }
- public int getView_num_ossean() {
- return view_num_ossean;
- }
- public void setView_num_ossean(int view_num_ossean) {
- this.view_num_ossean = view_num_ossean;
- }
- public String getHomepage() {
- return homepage;
- }
- public void setHomepage(String homepage) {
- this.homepage = homepage;
- }
- public String getUpdated_time() {
- return updated_time;
- }
- public void setUpdated_time(String updated_time) {
- this.updated_time = updated_time;
- }
- public String getExtracted_time() {
- return extracted_time;
- }
- public void setExtracted_time(String extracted_time) {
- this.extracted_time = extracted_time;
- }
- public int getComposite_score() {
- return composite_score;
- }
- public void setComposite_score(int composite_score) {
- this.composite_score = composite_score;
- }
- public int getRelative_memos_num() {
- return relative_memos_num;
- }
- public void setRelative_memos_num(int relative_memos_num) {
- this.relative_memos_num = relative_memos_num;
- }
- public String getCreated_time() {
- return created_time;
- }
- public void setCreated_time(String created_time) {
- this.created_time = created_time;
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/JobRequirement.java b/gather_program/src/main/java/org/ossean/gather/model/JobRequirement.java
deleted file mode 100644
index 9a4ea6940..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/JobRequirement.java
+++ /dev/null
@@ -1,212 +0,0 @@
-package org.ossean.gather.model;
-
-public class JobRequirement {
- private int id = 0;
- private String title = null;
- private String content = null;
- private String created_time = null;
- private String type = null;
- private String tags = null;
- private String url = null;
- private String url_md5 = null;
- private String author = null;
- private String author_url = null;
- private int view_num = 0;
- private int review_num = 0;
- private String extracted_time = null;
- private String source = null;
- private String similar_position = null;
- private String work_place = null;
- private String experience = null;
- private String salary = null;
- private String scale = null;
- private String domain = null;
- private String finance = null;
- private String style = null;
- private String education = null;
-
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
- public String getTitle() {
- return title;
- }
-
- public void setTitle(String title) {
- this.title = title;
- }
-
- public String getContent() {
- return content;
- }
-
- public void setContent(String content) {
- this.content = content;
- }
-
- public String getCreated_time() {
- return created_time;
- }
-
- public void setCreated_time(String created_time) {
- this.created_time = created_time;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getTags() {
- return tags;
- }
-
- public void setTags(String tags) {
- this.tags = tags;
- }
-
- public String getUrl() {
- return url;
- }
-
- public void setUrl(String url) {
- this.url = url;
- }
-
- public String getUrl_md5() {
- return url_md5;
- }
-
- public void setUrl_md5(String url_md5) {
- this.url_md5 = url_md5;
- }
-
- public String getAuthor() {
- return author;
- }
-
- public void setAuthor(String author) {
- this.author = author;
- }
-
- public String getAuthor_url() {
- return author_url;
- }
-
- public void setAuthor_url(String author_url) {
- this.author_url = author_url;
- }
-
- public int getView_num() {
- return view_num;
- }
-
- public void setView_num(int view_num) {
- this.view_num = view_num;
- }
-
- public int getReview_num() {
- return review_num;
- }
-
- public void setReview_num(int review_num) {
- this.review_num = review_num;
- }
-
- public String getExtracted_time() {
- return extracted_time;
- }
-
- public void setExtracted_time(String extracted_time) {
- this.extracted_time = extracted_time;
- }
-
- public String getSource() {
- return source;
- }
-
- public void setSource(String source) {
- this.source = source;
- }
-
- public String getSimilar_position() {
- return similar_position;
- }
-
- public void setSimilar_position(String similar_position) {
- this.similar_position = similar_position;
- }
-
- public String getWork_place() {
- return work_place;
- }
-
- public void setWork_place(String work_place) {
- this.work_place = work_place;
- }
-
- public String getExperience() {
- return experience;
- }
-
- public void setExperience(String experience) {
- this.experience = experience;
- }
-
- public String getSalary() {
- return salary;
- }
-
- public void setSalary(String salary) {
- this.salary = salary;
- }
-
- public String getScale() {
- return scale;
- }
-
- public void setScale(String scale) {
- this.scale = scale;
- }
-
- public String getDomain() {
- return domain;
- }
-
- public void setDomain(String domain) {
- this.domain = domain;
- }
-
- public String getFinance() {
- return finance;
- }
-
- public void setFinance(String finance) {
- this.finance = finance;
- }
-
- public String getStyle() {
- return style;
- }
-
- public void setStyle(String style) {
- this.style = style;
- }
-
- public String getEducation() {
- return education;
- }
-
- public void setEducation(String education) {
- this.education = education;
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/PKControlPosts.java b/gather_program/src/main/java/org/ossean/gather/model/PKControlPosts.java
deleted file mode 100644
index 08dfe0ceb..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/PKControlPosts.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package org.ossean.gather.model;
-
-public class PKControlPosts {
-
- private int id; //自增id用于生成固定项目固定的id
- private String urlMD5;
- public int getId() {
- return id;
- }
- public void setId(int id) {
- this.id = id;
- }
- public String getUrlMD5() {
- return urlMD5;
- }
- public void setUrlMD5(String urlMD5) {
- this.urlMD5 = urlMD5;
- }
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/PKControlProjects.java b/gather_program/src/main/java/org/ossean/gather/model/PKControlProjects.java
deleted file mode 100644
index 22c2ac224..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/PKControlProjects.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package org.ossean.gather.model;
-
-public class PKControlProjects {
-
- private int id; //自增id用于生成固定项目固定的id
- private String urlMD5;
- public int getId() {
- return id;
- }
- public void setId(int id) {
- this.id = id;
- }
- public String getUrlMD5() {
- return urlMD5;
- }
- public void setUrlMD5(String urlMD5) {
- this.urlMD5 = urlMD5;
- }
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/RelativeMemo.java b/gather_program/src/main/java/org/ossean/gather/model/RelativeMemo.java
deleted file mode 100644
index d39c4b2c1..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/RelativeMemo.java
+++ /dev/null
@@ -1,119 +0,0 @@
-package org.ossean.gather.model;
-
-public class RelativeMemo {
-
- private int id;
- private String title;
- private String content;
- private String created_time;
- private String updated_time;
- private String memo_type;
- private String tags;
- private String source;
- private String url;
- private String url_md5;
- private String author;
- private String author_url;
- private int view_num;
- private int review_num;
- private int view_num_ossean;
- private String extracted_time;
- public int getId() {
- return id;
- }
- public void setId(int id) {
- this.id = id;
- }
- public String getTitle() {
- return title;
- }
- public void setTitle(String title) {
- this.title = title;
- }
- public String getContent() {
- return content;
- }
- public void setContent(String content) {
- this.content = content;
- }
- public String getCreated_time() {
- return created_time;
- }
- public void setCreated_time(String created_time) {
- this.created_time = created_time;
- }
- public String getUpdated_time() {
- return updated_time;
- }
- public void setUpdated_time(String updated_time) {
- this.updated_time = updated_time;
- }
-
- public String getMemo_type() {
- return memo_type;
- }
- public void setMemo_type(String memo_type) {
- this.memo_type = memo_type;
- }
- public String getTags() {
- return tags;
- }
- public void setTags(String tags) {
- this.tags = tags;
- }
- public String getUrl() {
- return url;
- }
- public void setUrl(String url) {
- this.url = url;
- }
- public String getUrl_md5() {
- return url_md5;
- }
- public void setUrl_md5(String url_md5) {
- this.url_md5 = url_md5;
- }
- public String getAuthor() {
- return author;
- }
- public void setAuthor(String author) {
- this.author = author;
- }
- public String getAuthor_url() {
- return author_url;
- }
- public void setAuthor_url(String author_url) {
- this.author_url = author_url;
- }
- public String getSource() {
- return source;
- }
- public void setSource(String source) {
- this.source = source;
- }
- public int getView_num() {
- return view_num;
- }
- public void setView_num(int view_num) {
- this.view_num = view_num;
- }
- public int getReview_num() {
- return review_num;
- }
- public void setReview_num(int review_num) {
- this.review_num = review_num;
- }
- public int getView_num_ossean() {
- return view_num_ossean;
- }
- public void setView_num_ossean(int view_num_ossean) {
- this.view_num_ossean = view_num_ossean;
- }
- public String getExtracted_time() {
- return extracted_time;
- }
- public void setExtracted_time(String extracted_time) {
- this.extracted_time = extracted_time;
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/Taggings.java b/gather_program/src/main/java/org/ossean/gather/model/Taggings.java
deleted file mode 100644
index ada4124ca..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/Taggings.java
+++ /dev/null
@@ -1,71 +0,0 @@
-package org.ossean.gather.model;
-
-public class Taggings {
-
- private int id;
- private int tag_id;
- private String taggable_type;
- private int taggable_id;
- private int tagger_id;
- private String tagger_type;
- private String context;
- private String created_at;
- private int disagree_num;
- public int getId() {
- return id;
- }
- public void setId(int id) {
- this.id = id;
- }
- public int getTag_id() {
- return tag_id;
- }
- public void setTag_id(int tag_id) {
- this.tag_id = tag_id;
- }
-
- public int getTaggable_id() {
- return taggable_id;
- }
- public void setTaggable_id(int taggable_id) {
- this.taggable_id = taggable_id;
- }
- public String getTaggable_type() {
- return taggable_type;
- }
- public void setTaggable_type(String taggable_type) {
- this.taggable_type = taggable_type;
- }
- public int getTagger_id() {
- return tagger_id;
- }
- public void setTagger_id(int tagger_id) {
- this.tagger_id = tagger_id;
- }
- public String getTagger_type() {
- return tagger_type;
- }
- public void setTagger_type(String tagger_type) {
- this.tagger_type = tagger_type;
- }
- public String getContext() {
- return context;
- }
- public void setContext(String context) {
- this.context = context;
- }
- public String getCreated_at() {
- return created_at;
- }
- public void setCreated_at(String created_at) {
- this.created_at = created_at;
- }
- public int getDisagree_num() {
- return disagree_num;
- }
- public void setDisagree_num(int disagree_num) {
- this.disagree_num = disagree_num;
- }
-
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/model/Tags.java b/gather_program/src/main/java/org/ossean/gather/model/Tags.java
deleted file mode 100644
index 04f8ba77f..000000000
--- a/gather_program/src/main/java/org/ossean/gather/model/Tags.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.ossean.gather.model;
-
-public class Tags {
-
- private int id;
- private String name;
- public int getId() {
- return id;
- }
- public void setId(int id) {
- this.id = id;
- }
- public String getName() {
- return name;
- }
- public void setName(String name) {
- this.name = name;
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/process/AppContext.java b/gather_program/src/main/java/org/ossean/gather/process/AppContext.java
deleted file mode 100644
index 08d6727b5..000000000
--- a/gather_program/src/main/java/org/ossean/gather/process/AppContext.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.ossean.gather.process;
-
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-
-public class AppContext {
-
- static ApplicationContext appContext = new ClassPathXmlApplicationContext(
- "classpath:/applicationContext*.xml");
-}
-
diff --git a/gather_program/src/main/java/org/ossean/gather/process/DataHandler.java b/gather_program/src/main/java/org/ossean/gather/process/DataHandler.java
deleted file mode 100644
index 2079d9108..000000000
--- a/gather_program/src/main/java/org/ossean/gather/process/DataHandler.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.ossean.gather.process;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class DataHandler {
-
- // 获取当前时间的String
- public static String getNow() {
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- String result = sdf.format(new Date());
- return result;
- }
-
- // 标签分离函数
- public static List tagsSegmentation(String tags) {
- List tag = new ArrayList();
-
- if (tags != null) {
- String regex = "<[^<>]*>";
- Pattern pattern = Pattern.compile(regex);
- Matcher matcher = pattern.matcher(tags);
-
- while (matcher.find()) {
- String t = matcher.group();
- t = t.substring(1, t.length() - 1);
-
- tag.add(t);
- }
- }
- return tag;
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/process/GatherProcess.java b/gather_program/src/main/java/org/ossean/gather/process/GatherProcess.java
deleted file mode 100644
index d4af98e07..000000000
--- a/gather_program/src/main/java/org/ossean/gather/process/GatherProcess.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.ossean.gather.process;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import javax.annotation.Resource;
-
-import org.apache.log4j.Logger;
-import org.ossean.gather.model.Configure;
-import org.ossean.gather.sourceDao.GatherDao;
-import org.ossean.gather.targetDao.PointerDao;
-import org.ossean.gather.targetDao.TargetDao;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.stereotype.Component;
-
-@Component
-public class GatherProcess {
- private static Logger logger = Logger.getLogger(GatherProcess.class);
- public static Map gatherState = new HashMap(); //
- public static Set urlMd5Set = new HashSet(); //所有gather数据url_md5放入内存
- private Configure conf;
- private ExecutorService pool = Executors.newFixedThreadPool(20);
- @Resource
- private GatherDao gatherDao;
- @Resource
- private TargetDao targetDao;
- @Resource
- private PointerDao pointerDao;
-
- public void start(String configureName){
- while(true){
- conf = new Configure(configureName);
- //urlMd5Set = targetDao.initUrlMd5Set(conf.getTargetTableName());
- for(String sourceTableName : conf.getSourceTableNameList()){
- Boolean state = gatherState.get(sourceTableName);
- if( state == null || state == false){
- gatherState.put(sourceTableName, true);
- }
- else if(state == true){
- continue;
- }
- //GatherThread gatherThread = (GatherThread)AppContext.appContext.getBean("gatherThread");
- GatherThreadNew gatherThreadNew = (GatherThreadNew)AppContext.appContext.getBean("gatherThreadNew");
- gatherThreadNew.setParameters(conf,sourceTableName);
- pool.execute(gatherThreadNew);
- }
- try {
- logger.info(".......sleeping......");
- Thread.sleep(20000);
- } catch (InterruptedException e) {
- logger.error(e);
- }
- System.gc(); //手动垃圾回收
- }
-
- }
-
- public static void main(String[] args) throws InterruptedException {
- //获取传递的参数
- String configureName = "";
- if (args.length != 0) {
- configureName = args[0].toString();
- } else {
- logger.error("404 configure!");
- configureName = "relative_memos";
- //configureName = "gather_projects";
- //return;
- }
- ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
- GatherProcess main = applicationContext.getBean(GatherProcess.class);
- main.start(configureName);
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/process/GatherThread.java b/gather_program/src/main/java/org/ossean/gather/process/GatherThread.java
deleted file mode 100644
index 9b691cb20..000000000
--- a/gather_program/src/main/java/org/ossean/gather/process/GatherThread.java
+++ /dev/null
@@ -1,363 +0,0 @@
-package org.ossean.gather.process;
-
-import java.util.List;
-
-import javax.annotation.Resource;
-
-import org.apache.log4j.Logger;
-import org.ossean.gather.model.Configure;
-import org.ossean.gather.model.GatherProject;
-import org.ossean.gather.model.JobRequirement;
-import org.ossean.gather.model.PKControlPosts;
-import org.ossean.gather.model.PKControlProjects;
-import org.ossean.gather.model.RelativeMemo;
-import org.ossean.gather.model.Taggings;
-import org.ossean.gather.sourceDao.GatherDao;
-import org.ossean.gather.sourceDao.PKControlPostsDao;
-import org.ossean.gather.sourceDao.PKControlProjectsDao;
-import org.ossean.gather.targetDao.PointerDao;
-import org.ossean.gather.targetDao.TargetDao;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-@Component("gatherThread")
-@Scope("prototype")
-public class GatherThread implements Runnable {
- private static Logger logger = Logger.getLogger(GatherThread.class);
- private Configure conf;
-
- @Resource
- private GatherDao gatherDao;
- @Resource
- private PointerDao pointerDao;
- @Resource
- private PKControlPostsDao pkControlPostsDao;
- @Resource
- private TargetDao targetDao;
- @Resource
- private PKControlProjectsDao pkControlProjectsDao;
-
- private int idsBegin; // 转移开始Id值
- private int idsEnd; // 转移结束Id值
- private int idsIncrement;// 每次转移的Id量
-
- private int beginId;
- private int endId;
-
- private String sourceTableName;
- private String pkControlPostsTableName = "pk_control_posts";
- private String pkControlProjectsTableName = "pk_control_projects";
- private String memoTaggingsTableName = "memo_taggings";
- private String tagsTableName = "tags";
-
- private String gatherPostsTableName = "relative_memos";
- private int maxId;
-
- public void setParameters(Configure conf, String sourceTableName) {
- this.conf = conf;
- this.sourceTableName = sourceTableName;
- }
-
- // 读指针
- public int readPointer(String table, String source, String target) {
- int pointer = 1;
- try {
- pointer = pointerDao.getPointer(table, source, target);
- } catch (Exception e) {
- // 表示表中没有数据
- logger.info("No such pointer! Create one");
- pointerDao.insertPointer(table, source, target, 1);
- }
- return pointer;
- }
-
- @Override
- public void run() {
-// long start = System.currentTimeMillis();
- Thread.currentThread().setName(sourceTableName);
- idsIncrement = conf.getIdsIncrement();
- idsBegin = readPointer(conf.getPointerTableName(), sourceTableName,
- conf.getTargetTableName());
- idsEnd = maxId = gatherDao.getMaxId(sourceTableName);
- while (idsBegin < idsEnd) {
- beginId = idsBegin;
- endId = beginId + idsIncrement - 1; // 取数据时两边都取等号
- if (endId <= maxId) {
- handleBatchData(beginId, endId, conf);
- idsBegin = idsBegin + idsIncrement;
- } else {
- endId = maxId; // endId应小于maxId
- handleBatchData(beginId, endId, conf);
- break;
- }
- }
- GatherProcess.gatherState.put(sourceTableName, false);
-// long end = System.currentTimeMillis();
-// logger.info((end - start) / 6000);
- }
-
- public void handleBatchData(int beginId, int endId, Configure conf) {
- logger.info("BeginId#" + sourceTableName + ":" + beginId);
- // 表示任务没有完成
- int maxId = gatherDao.getMaxId(sourceTableName);
- // 防止转移超过当前最大值的Id数据
- if (beginId >= 0 && endId > 0 && maxId >= endId) {
- // 更新执行开始时间
- logger.info("begin gathering...");
- // 插入Id段数据,忽略重复值
- try {
- String[] sourceFields = conf.getSourceFields().split(",");
- String[] targetFields = conf.getTargetFields().split(",");
- String selectItems = "";
- for (int i = 0; i < sourceFields.length; i++) {
- String str_source = sourceFields[i];
- String str_target = targetFields[i];
- selectItems += str_source + " as " + str_target + ",";
- }
- selectItems = selectItems
- .substring(0, selectItems.length() - 1) + " ";
- if (conf.getTargetTableName().equals("relative_memos")) {
- List dataGet = gatherDao.getPostGatherData(
- sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- for (int i = 0; i < dataGet.size(); i++) {
- RelativeMemo model = dataGet.get(i);
- String urlMD5 = model.getUrl_md5();// 通过urlMD5判断是不是已经存在该帖子
- // 是否更新
- int postId = 0;
- if(GatherProcess.urlMd5Set.contains(urlMD5)){
- //urlmd5存在则更新
- RelativeMemo samePost = targetDao.findPostByUrlMD5(
- conf.getTargetTableName(), urlMD5);
- // update gather_projects表中对应的记录,在维持待更新表
- postId = samePost.getId();
- model.setId(postId);
- handleUpdateGatherPosts(samePost.getId(), model);
- }else{
- // 不存在 插入
- PKControlPosts pkControlModel = pkControlPostsDao
- .selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);// 查看有没有固定的id
- if (pkControlModel != null)
- model.setId(pkControlModel.getId());
- else {
- // 在pk_control_posts表中生成当前项目对应的id
- pkControlPostsDao.insertOneItem(
- pkControlPostsTableName, urlMD5);
- // 查看刚刚插入信息的id
- PKControlPosts controlItem = pkControlPostsDao
- .selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);
- // 用id构造model对应的固定不变的id
- model.setId(controlItem.getId());
- postId = model.getId();
- }
- handleInsertGatherPosts(model, conf);
- GatherProcess.urlMd5Set.add(urlMD5);
- }
-
- // 将tag和项目的关系存入表item_tag_relation 并分离tag
- String tags = model.getTags();
- if (tags == null) {
- // 表示该项目没有标签
- continue;
- }
- List tagList = DataHandler
- .tagsSegmentation(tags);
- for (String tag : tagList) {
- targetDao.insertTag(tagsTableName, tag);// ignore方式插入该项目的标签
- int tag_id = targetDao.selectTagIdByName(
- tagsTableName, tag);
- Taggings taggings = new Taggings();
- taggings.setTag_id(tag_id);
- taggings.setTaggable_id(postId);
- taggings.setTaggable_type("RelativeMemo");
- taggings.setContext("tags");
- taggings.setCreated_at(DataHandler.getNow());
- // 将Taggings对象存入数据库中
- try {
- targetDao.insertTaggings(memoTaggingsTableName,
- taggings);
- } catch (Exception e) {
- // 在插入记录之前 relative_memos表中的记录已经被删除掉了
- logger.error(e);
- System.exit(0);
- }
- }
- }
- } else if (conf.getTargetTableName().equals("gather_projects")) {
- List dataGet = gatherDao.getPrjGatherData(
- sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- for (int i = 0; i < dataGet.size(); i++) {
- GatherProject model = dataGet.get(i);
- String urlMD5 = model.getUrl_md5();// 通过urlMD5判断是不是已经存在该项目
- // 是否更新
- int prjId = 0;
- if(GatherProcess.urlMd5Set.contains(urlMD5)){
- GatherProject samePrj = targetDao.findPrjByUrlMD5(
- conf.getTargetTableName(), urlMD5);
- // update gather_projects表中对应的记录,在维持待更新表
- prjId = samePrj.getId();
- model.setId(prjId);
- model.setUpdate_mark(2);
- handleUpdateGatherProjects(samePrj.getId(), model);
- }else{
- // 不存在 插入
- PKControlProjects pkControlProjects = pkControlProjectsDao
- .selectItemByUrlMD5(
- pkControlProjectsTableName, urlMD5);// 查看有没有固定的id
- if (pkControlProjects != null)
- model.setId(pkControlProjects.getId());
- else {
- // 在pk_control_posts表中生成当前项目对应的id
- pkControlProjectsDao.insertOneItem(
- pkControlProjectsTableName, urlMD5);
- // 查看刚刚插入信息的id
- PKControlProjects controlItem = pkControlProjectsDao
- .selectItemByUrlMD5(
- pkControlProjectsTableName,
- urlMD5);
- // 用id构造model对应的固定不变的id
- model.setId(controlItem.getId());
- prjId = model.getId();
- }
- model.setUpdate_mark(0);
- handleInsertGatherProjects(model, conf);
- GatherProcess.urlMd5Set.add(urlMD5);
- }
-
-// // 将tag和项目的关系存入表item_tag_relation 并分离tag
-// String tags = model.getTags();
-// if (tags == null) {
-// // 表示该项目没有标签
-// continue;
-// }
-// List tagList = DataHandler
-// .tagsSegmentation(tags);
-// for (String tag : tagList) {
-// targetDao.insertTag(tagsTableName, tag);// ignore方式插入该项目的标签
-// int tag_id = targetDao.selectTagIdByName(
-// tagsTableName, tag);
-// Taggings taggings = new Taggings();
-// taggings.setTag_id(tag_id);
-// taggings.setTaggable_id(prjId);
-// taggings.setTaggable_type("OpenSourceProject");
-// taggings.setContext("tags");
-// taggings.setCreated_at(DataHandler.getNow());
-// // 将Taggings对象存入数据库中
-// try {
-// targetDao.insertTaggings(taggingsTableName,
-// taggings);
-// } catch (Exception e) {
-// // 在插入记录之前 relative_memos表中的记录已经被删除掉了
-// logger.error(e);
-// System.exit(0);
-// }
-// }
- }
- } else {
- List dataGet = gatherDao.getJobGatherData(
- sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- for (int i = 0; i < dataGet.size(); i++) {
- JobRequirement model = dataGet.get(i);
- String urlMD5 = model.getUrl_md5();// 通过urlMD5判断是不是已经存在该帖子
- // 是否更新
- int postId = 0;
- if(GatherProcess.urlMd5Set.contains(urlMD5)){
- JobRequirement sameJob = targetDao.findJobByUrlMD5(
- conf.getTargetTableName(), urlMD5);
- // update gather_projects表中对应的记录,在维持待更新表
- postId = sameJob.getId();
- model.setId(postId);
- handleUpdateGatherJobs(sameJob.getId(), model);
- }else{
- // 不存在 插入
- PKControlPosts pkControlPosts = pkControlPostsDao
- .selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);// 查看有没有固定的id
- if (pkControlPosts != null)
- model.setId(pkControlPosts.getId());
- else {
- // 在pk_control_posts表中生成当前项目对应的id
- pkControlPostsDao.insertOneItem(
- pkControlPostsTableName, urlMD5);
- // 查看刚刚插入信息的id
- PKControlPosts controlItem = pkControlPostsDao
- .selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);
- // 用id构造model对应的固定不变的id
- model.setId(controlItem.getId());
- postId = model.getId();
- }
- handleInsertGatherJobs(model, conf);
- GatherProcess.urlMd5Set.add(urlMD5);
- }
- }
- }
-
- } catch (Exception ex) {
- // 数据迁移过程可能发生异常情况
- logger.error(ex);
- System.exit(0);
- }
-
- // 更新游标到本次 EndId+1;
- pointerDao.updatePointer(conf.getPointerTableName(),
- sourceTableName, conf.getTargetTableName(), endId + 1);// sourceIdBegin
- // +
- // idsIncrement
- logger.info("current--" + sourceTableName + ": " + endId);
- }
- }
-
- // 处理URL不存在的帖子 插入relative_memos表
- public void handleInsertGatherPosts(RelativeMemo model, Configure conf) {
- try {
- targetDao.insertRelativeMemo(conf.getTargetTableName(),
- conf.getTargetFields(), model);
- } catch (Exception e) {
- logger.error(e);
- }
- }
-
- // 处理URL相同的帖子更新 id表示更新的帖子固定id
- public void handleUpdateGatherPosts(int id, RelativeMemo model_new) {
- targetDao.updateRelativeMemo(gatherPostsTableName, model_new, id);// 更新数据relative_memos表
- }
-
- // 处理URL不存在的项目 插入gather_projects表
- public void handleInsertGatherProjects(GatherProject model, Configure conf) {
- try {
- targetDao.insertOpenSourceProject(conf.getTargetTableName(),
- conf.getTargetFields(), model);
- } catch (Exception e) {
- logger.error(e);
- }
-
- }
-
- // 处理URL相同的项目更新 id表示更新的项目固定id
- public void handleUpdateGatherProjects(int id, GatherProject model_new) {
- targetDao.updateOpenSourceProject(conf.getTargetTableName(), model_new,
- id);// 更新数据gather_projects表
- }
-
- // 处理URL不存在的项目 插入job_requirements表
- public void handleInsertGatherJobs(JobRequirement model, Configure conf) {
- try {
- targetDao.insertJobRequirement(conf.getTargetTableName(),
- conf.getTargetFields(), model);
- } catch (Exception e) {
- logger.error(e);
- }
-
- }
-
- // 处理URL相同的项目更新 id表示更新的项目固定id
- public void handleUpdateGatherJobs(int id, JobRequirement model_new) {
- targetDao
- .updateJobRequirement(conf.getTargetTableName(), model_new, id);// 更新数据job_requirements表
- }
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/process/GatherThreadNew.java b/gather_program/src/main/java/org/ossean/gather/process/GatherThreadNew.java
deleted file mode 100644
index c1c4253a6..000000000
--- a/gather_program/src/main/java/org/ossean/gather/process/GatherThreadNew.java
+++ /dev/null
@@ -1,512 +0,0 @@
-package org.ossean.gather.process;
-/**
- * @author 李乾坤 2016年12月5号第一次修改。修改内容如下:
- * 1.将获取tag部分抽离为一个单独的可以接收参数的函数handleTags
- * 2.改变了数据汇总策略。放弃之前把记录全放到set中的方法。采用如下策略:
- * 1>查询pk_control_*表,查看当前帖子或者项目是否已经有固定Id
- * 2>查询到固定Id值(sameId),则依据sameId更新数据库中的内容。未查到固定Id进行3>
- * 3>将当前项目或帖子依据url_md5插入pk_control_*表以生成固定Id(newId),然后根据newId执行插入操作。
- * */
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.Resource;
-
-import org.apache.log4j.Logger;
-import org.ossean.gather.model.Configure;
-import org.ossean.gather.model.GatherProject;
-import org.ossean.gather.model.JobRequirement;
-import org.ossean.gather.model.PKControlPosts;
-import org.ossean.gather.model.PKControlProjects;
-import org.ossean.gather.model.RelativeMemo;
-import org.ossean.gather.model.Taggings;
-import org.ossean.gather.sourceDao.GatherDao;
-import org.ossean.gather.sourceDao.PKControlPostsDao;
-import org.ossean.gather.sourceDao.PKControlProjectsDao;
-import org.ossean.gather.targetDao.PointerDao;
-import org.ossean.gather.targetDao.TargetDao;
-import org.springframework.context.annotation.Scope;
-import org.springframework.jmx.export.annotation.ManagedResource;
-import org.springframework.stereotype.Component;
-import org.springframework.transaction.annotation.Propagation;
-import org.springframework.transaction.annotation.Transactional;
-
-@Component("gatherThreadNew")
-@Scope("prototype")
-public class GatherThreadNew implements Runnable {
- private static Logger logger = Logger.getLogger(GatherThread.class);
- private Configure conf;
-
- //注解的方式生成bean并获取汇总操作对象
- @Resource
- private GatherDao gatherDao;
- @Resource
- private PointerDao pointerDao;
- //控制帖子的操作
- @Resource
- private PKControlPostsDao pkControlPostsDao;
- //汇总表操作对象
- @Resource
- private TargetDao targetDao;
- //项目控制表操作
- @Resource
- private PKControlProjectsDao pkControlProjectsDao;
-
- private int idsBegin; // 转移开始Id值
- private int idsEnd; // 转移结束Id值
- private int idsIncrement;// 每次转移的Id量
-
- private int beginId;
- private int endId;
-
- private String sourceTableName;
- private String pkControlPostsTableName = "pk_control_posts";
- private String pkControlProjectsTableName = "pk_control_projects";
-
- private String gatherPostsTableName = "relative_memos";
- private int maxId;
-
- private String memoTaggingsTableName = "memo_taggings";
- private String tagsTableName = "tags";
-
- public void setParameters(Configure conf, String sourceTableName) {
- this.conf = conf;
- this.sourceTableName = sourceTableName;
- }
-
- // 读指针
- public int readPointer(String table, String source, String target) {
- int pointer = 1;
- try {
- //调用pointerDao对象读取数据库的pointer表获取上次汇总的位置
- pointer = pointerDao.getPointer(table, source, target);
- } catch (Exception e) {
- // 表示表中没有数据
- logger.info("No such pointer! Create one");
- //表不存在的话就调用pointerDao中的函数创建pointer的表并插入1
- pointerDao.insertPointer(table, source, target, 1);
- }
- return pointer;
- }
-
- @Override
- public void run(){
- //获取系统当年时间
- //long start = System.currentTimeMillis();
-
- //设置当前进程数据源的表名
- Thread.currentThread().setName(sourceTableName);
- //idscrement为一个批处理的处理的数量即id一次增加的数量
- idsIncrement = conf.getIdsIncrement();
- //读取上次汇总结束的地址做为本次汇总开始的地址
- idsBegin = readPointer(conf.getPointerTableName(), sourceTableName,
- conf.getTargetTableName());
- //读取数据来源表的数量
- idsEnd = maxId = gatherDao.getMaxId(sourceTableName);
- //idsEnd = maxId = 1000;
- //如果开始的ID小于结束的id就从idsBegin开始汇总
- while (idsBegin < idsEnd) {
- beginId = idsBegin;
- endId = beginId + idsIncrement - 1; // 取数据时两边都取等号
- if (endId <= maxId) {
- //如果上一批量结尾的id小于最大id说明汇总还没完成,就继续进行
- handleBatchData(beginId, endId, conf);
- idsBegin = idsBegin + idsIncrement;
- } else {
- endId = maxId; // endId应小于maxId
- handleBatchData(beginId, endId, conf);
- break;
- }
- }
- GatherProcess.gatherState.put(sourceTableName, false);
-
- }
-
- @Transactional(propagation=Propagation.REQUIRED)
- public void handleBatchData(int beginId, int endId, Configure conf) {
- logger.info("BeginId#" + sourceTableName + ":" + beginId);
- // 表示任务没有完成
- int maxId = gatherDao.getMaxId(sourceTableName);
- // 防止转移超过当前最大值的Id数据
- if (beginId >= 0 && endId > 0 && maxId >= endId) {
- // 更新执行开始时间
- logger.info("begin gathering...");
-
- // 插入Id段数据,忽略重复值
- try {
- logger.info("准备查询批量数据");
- String selectItems = getSelectItems(conf);
- //这里的目标表是relative_memos
- if (conf.getTargetTableName().equals("relative_memos")) {
- logger.info("对帖子进行汇总");
-
- List dataGet = gatherDao.getPostGatherData(sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- logger.info("从数据源表中提取到数据");
- gatherPosts(dataGet);
-
- } else
- //这是对项目进行汇总的操作
- if (conf.getTargetTableName().equals("gather_projects")) {
- List dataGet = null;
- if(sourceTableName.equals("github"))
- dataGet = gatherDao.getPrjGatherDataForGit(sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- else
- dataGet = gatherDao.getPrjGatherData(sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- gatherProjects(dataGet);
-
- } else {
- List dataGet = gatherDao.getJobGatherData(
- sourceTableName, selectItems, beginId, endId,
- conf.getAndWhere());
- gatehrRequirement(dataGet);
- }
-
- } catch (Exception ex) {
- // 数据迁移过程可能发生异常情况
- ex.printStackTrace();
- logger.error(ex);
- System.exit(0);
- }
-
- // 更新游标到本次 EndId+1;
- pointerDao.updatePointer(conf.getPointerTableName(),
- sourceTableName, conf.getTargetTableName(), endId + 1);// sourceIdBegin
- // +
- // idsIncrement
- logger.info("current--" + sourceTableName + ": " + endId);
- }
- }
-
- // 处理URL不存在的帖子 插入relative_memos表
- public void handleInsertGatherPosts(RelativeMemo model, Configure conf) {
- try {
- targetDao.insertRelativeMemo(conf.getTargetTableName(),
- conf.getTargetFields(), model);
- } catch (Exception e) {
- logger.error(e);
- }
- }
-
- // 处理URL相同的帖子更新 id表示更新的帖子固定id//更新url_md5相同的帖子
- public void handleUpdateGatherPosts(int id, RelativeMemo model_new) {
- targetDao.updateRelativeMemo(gatherPostsTableName, model_new, id);// 更新数据relative_memos表
- }
- public void handleUpdateGatherPostsByUrlMD5(int id, RelativeMemo model_new) {
- targetDao.updateRelativeMemoByUrlMD5(gatherPostsTableName, model_new, id);// 更新数据relative_memos表
- }
-
- // 处理URL不存在的项目 插入gather_projects表
- public void handleInsertGatherProjects(GatherProject model, Configure conf) {
- try {
- targetDao.insertOpenSourceProject(conf.getTargetTableName(),
- conf.getTargetFields(), model);
- } catch (Exception e) {
- logger.error(e);
- }
-
- }
-
- // 处理URL相同的项目更新 id表示更新的项目固定id
- public void handleUpdateGatherProjects(int id, GatherProject model_new) {
- targetDao.updateOpenSourceProject(conf.getTargetTableName(), model_new,
- id);// 更新数据gather_projects表
- }
-
- // 处理URL不存在的项目 插入job_requirements表
- public void handleInsertGatherJobs(JobRequirement model, Configure conf) {
- try {
- targetDao.insertJobRequirement(conf.getTargetTableName(),
- conf.getTargetFields(), model);
- } catch (Exception e) {
- logger.error(e);
- }
-
- }
-
- // 处理URL相同的项目更新 id表示更新的项目固定id
- public void handleUpdateGatherJobs(int id, JobRequirement model_new) {
- targetDao.updateJobRequirement(conf.getTargetTableName(), model_new, id);// 更新数据job_requirements表
- }
-
- // 将tag和项目的关系存入表item_tag_relation 并分离tag
- public void handleTags(RelativeMemo model,int postId)
- {
- // 将tag和项目的关系存入表item_tag_relation 并分离tag
- String tags = model.getTags();
- if(tags != null)
- {
- List tagList = DataHandler.tagsSegmentation(tags);
- for (String tag : tagList) {
- targetDao.insertTag(tagsTableName, tag);// ignore方式插入该项目的标签
- int tag_id = targetDao.selectTagIdByName(
- tagsTableName, tag);
- Taggings taggings = new Taggings();
- taggings.setTag_id(tag_id);
- taggings.setTaggable_id(postId);
- taggings.setTaggable_type("RelativeMemo");
- taggings.setContext("tags");
- taggings.setCreated_at(DataHandler.getNow());
- // 将Taggings对象存入数据库中
- try {
- targetDao.insertTaggings(memoTaggingsTableName,
- taggings);
- } catch (Exception e) {
- // 在插入记录之前 relative_memos表中的记录已经被删除掉了
- logger.error(e);
- System.exit(0);
- }
- }
- }
- }
-
- /**
- * 最新修改的程序更好地考虑到了程序的有效性,主要分以下情况:
- * 1.pk_control_posts中已存在(查询到pk_model),则先根据url_md5查询targetTable中是否存在改对象(设为model)。
- * 如果model不为空,则判断pk_model.getId == model.getId是否成立,成立则直接更新,不成立则将pk_control中的
- * 更新为model.getId.
- * 2.pk_control_posts中不存在,则直接插入生成对象并再次查询到pk_model,先根据url_md5查询targetTable中是否存在改对象(设为model)。
- * 如果model不为空,则判断pk_model.getId == model.getId是否成立,成立则直接更新,不成立则将pk_control中的
- * 更新为model.getId.
- * @param dataGet
- */
- @Transactional(propagation=Propagation.REQUIRED)
- public void gatherPosts(List dataGet){
- for (int i = 0; i < dataGet.size(); i++) {
- RelativeMemo model = dataGet.get(i);
- String urlMD5 = model.getUrl_md5();// 通过urlMD5判断是不是已经存在该帖子、是否更新
-
- int postId = 0;
- logger.info("开始汇总:" + model.getId());
-
- //查找pk_control_posts表以判定此帖子是否已经存在
- PKControlPosts pkControlModel = pkControlPostsDao.selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);
- //帖子已经存在则进行更新操作
- if(pkControlModel != null){
-
- try{
- logger.info("帖子已经在pk_control_posts存在");
- RelativeMemo memosModel = targetDao.selectRelativeMemosItemByUrlMD5(conf.getTargetTableName(), model.getUrl_md5());
- if(memosModel!=null)
- {
- logger.info("帖子确实已经插入到了relative_memos表中");
- try
- {
- if(memosModel.getId() == pkControlModel.getId())
- {
- logger.error("memosModel.getId() == pkControlModel.getId()");
- model.setId(pkControlModel.getId());
- handleUpdateGatherPosts(pkControlModel.getId(), model);
- }
- else
- {
- logger.error("memosModel.getId() != pkControlModel.getId()");
- gatherDao.updatePkControlTable(pkControlPostsTableName,
- memosModel.getId(), memosModel.getUrl_md5());
- pkControlModel.setId(model.getId());
- model.setId(memosModel.getId());
- targetDao.deleteWhileExist(gatherPostsTableName,model);
- handleUpdateGatherPostsByUrlMD5(pkControlModel.getId(), model);
- }
-
- logger.info("更新完成");
- }
- catch(Exception e)
- {
- logger.error(e);
- logger.error("更新失败:295");
- System.exit(0);
- }
- }
- else
- {
- logger.info("帖子实际上并未插入到relative_memos表中");
- try{
- model.setId(pkControlModel.getId());
- targetDao.deleteWhileExist(gatherPostsTableName,model);
- handleInsertGatherPosts(model, conf);
- logger.info("更新转为插入");
- }
- catch (Exception e)
- {
- logger.error("更新转为插入时出错:298");
- System.exit(0);
-
- }
- }
-
- }catch(Exception e){
- logger.error("更新帖子时出错:" + e);
- System.exit(0);
- }
- }
- else
- {//帖子不存在就插入pk_control_posts表并为帖子生成唯一固定ID
- logger.info("准备将帖子插入pk_control_posts");
- try{
- pkControlPostsDao.insertOneItem(pkControlPostsTableName, urlMD5);
-
- }catch(Exception e)
- {
- logger.info("插入pkcontorl时失败");
- //System.exit(0);
- continue;
- }
- PKControlPosts controlItem = pkControlPostsDao.selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);
-
- try{
- RelativeMemo memosModel2 = targetDao.selectRelativeMemosItemByUrlMD5(conf.getTargetTableName(), model.getUrl_md5());
-
- if(memosModel2 == null)
- {
- model.setId(controlItem.getId());
- targetDao.deleteWhileExist(gatherPostsTableName,model);
- handleInsertGatherPosts(model, conf);
- }
- else
- {
- if(controlItem.getId() == memosModel2.getId())
- {
- model.setId(controlItem.getId());
- int sucess2 = targetDao.updateRelativeMemoByUrlMD5(gatherPostsTableName, model, model.getId());
- if(sucess2 != 1)
- {
- logger.error("365");
- System.exit(0);
- }
- }
- else
- {
- gatherDao.updatePkControlTable(pkControlPostsTableName, memosModel2.getId(), memosModel2.getUrl_md5());
- model.setId(memosModel2.getId());
- controlItem.setId(memosModel2.getId());
- targetDao.deleteWhileExist(gatherPostsTableName,model);
- handleUpdateGatherPosts(controlItem.getId(), model);
-
- }
-
- }
- logger.info("插入relative_memos成功");
- }catch(Exception e){
- logger.error("插入帖子时出错:" + e);
- int sucess = targetDao.updateRelativeMemoByUrlMD5(gatherPostsTableName, model, model.getId());
- if(sucess != 1)System.exit(0);
- }
- }
-
- postId = model.getId();
- // 将tag和项目的关系存入表item_tag_relation 并分离tag
- handleTags(model,postId);
-
- }
- }
- @Transactional(propagation=Propagation.REQUIRED)
- public void gatherProjects(List dataGet){
- for (int i = 0; i < dataGet.size(); i++) {
- GatherProject model = dataGet.get(i);
- String urlMD5 = model.getUrl_md5();// 通过urlMD5判断是不是已经存在该项目、是否更新
-
- int prjId = 0;
-
- PKControlProjects pkControlProjects = pkControlProjectsDao.selectItemByUrlMD5(
- pkControlProjectsTableName, urlMD5);//查看有没有固定的id
-
- if(pkControlProjects != null){
- model.setId(pkControlProjects.getId());
- model.setUpdate_mark(1);
-
- try{
- if(targetDao.selectOpenSourceProjectsItem(conf.getTargetTableName(), model.getId())!=null)
- handleUpdateGatherProjects(pkControlProjects.getId(), model);
- else{
- model.setUpdate_mark(0);
- handleInsertGatherProjects(model, conf);
- }
- }catch(Exception e){
- logger.error("更新项目时出错:" + e);
- }
-
- }else{
- // 在pk_control_posts表中生成当前项目对应的id
- pkControlProjectsDao.insertOneItem(
- pkControlProjectsTableName, urlMD5);
- // 查看刚刚插入信息的id
- PKControlProjects controlItem = pkControlProjectsDao.selectItemByUrlMD5(
- pkControlProjectsTableName,urlMD5);
- // 用id构造model对应的固定不变的id
- model.setId(controlItem.getId());
-
- model.setUpdate_mark(0);
- try{
- handleInsertGatherProjects(model, conf);
- }catch(Exception e){
- logger.error("插入项目时出错:" + e);
- }
- }
-
- prjId = model.getId();
- }
- }
-
- @Transactional(propagation=Propagation.REQUIRED)
- public void gatehrRequirement(List dataGet){
- for (int i = 0; i < dataGet.size(); i++) {
- JobRequirement model = dataGet.get(i);
- String urlMD5 = model.getUrl_md5();// 通过urlMD5判断是不是已经存在该帖子
- // 是否更新
- int postId = 0;
-
- PKControlPosts pkControlPosts = pkControlPostsDao
- .selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);// 查看有没有固定的id
-
- if (pkControlPosts != null){
- model.setId(pkControlPosts.getId());
-
- try{
- handleUpdateGatherJobs(pkControlPosts.getId(), model);
- }catch(Exception e){
- logger.error("更新jobRequirement时出错:" + e);
- }
- }
- else {
- // 在pk_control_posts表中生成当前项目对应的id
- pkControlPostsDao.insertOneItem(
- pkControlPostsTableName, urlMD5);
- // 查看刚刚插入信息的id
- PKControlPosts controlItem = pkControlPostsDao
- .selectItemByUrlMD5(
- pkControlPostsTableName, urlMD5);
- // 用id构造model对应的固定不变的id
- model.setId(controlItem.getId());
-
- try{
- handleInsertGatherJobs(model, conf);
- }catch(Exception e){
- logger.error("插入jobRequirement时出错:"+ e);
- }
- }
- postId = model.getId();
- }
- }
-
- public String getSelectItems(Configure conf){
- String[] sourceFields = conf.getSourceFields().split(",");
- String[] targetFields = conf.getTargetFields().split(",");
- String selectItems = "";
- for (int i = 0; i < sourceFields.length; i++) {
- String str_source = sourceFields[i];
- String str_target = targetFields[i];
- selectItems += str_source + " as " + str_target + ",";
- }
- selectItems = selectItems
- .substring(0, selectItems.length() - 1) + " ";
- return selectItems;
- }
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/sourceDao/GatherDao.java b/gather_program/src/main/java/org/ossean/gather/sourceDao/GatherDao.java
deleted file mode 100644
index d8a5ee523..000000000
--- a/gather_program/src/main/java/org/ossean/gather/sourceDao/GatherDao.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package org.ossean.gather.sourceDao;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-import org.ossean.gather.model.GatherProject;
-import org.ossean.gather.model.JobRequirement;
-import org.ossean.gather.model.RelativeMemo;
-
-public interface GatherDao {
-
- // 读取表中最大id值
- @Select("select max(id) from ${table}")
- public int getMaxId(@Param("table") String table);
-
- // 向gather_tasks表插入转移任务
- @Insert("INSERT ignore INTO ${taskTableName} (BeginId,EndId,SourceTableName,TargetTableName) VALUES (${idsBegin},${idsEnd},#{sourceTableName},#{targetTableName})")
- public int insertLog(@Param("idsBegin") int idsBegin,
- @Param("idsEnd") int idsEnd,
- @Param("taskTableName") String taskTableName,
- @Param("sourceTableName") String sourceTableName,
- @Param("targetTableName") String targetTableName);
-
- /**
- * 更新pk_control_posts
- * @param targettable
- * @param id
- * @param url_md5
- */
- @Update("update ${targettable} set id = #{id} where url_md5 = #{url_md5}")
- public void updatePkControlTable(@Param("targettable")String targettable,@Param("id")int id,
- @Param("url_md5")String url_md5);
- // 查询未处理的task
- @Select("SELECT BeginId,EndId,BeginTime,EndTime "
- + "FROM ${migrationTask} "
- + "WHERE IsDone = 0 AND SourceTableName=#{sourceTableName} AND TargetTableName=#{targetTableName} ORDER BY BeginId LIMIT 0,1")
- public Map checkDone(@Param("migrationTask") String migrationTask,
- @Param("sourceTableName") String sourceTableName,
- @Param("targetTableName") String targetTableName);
-
- // 更新转移任务开始时间
- @Update("UPDATE ${migrationTask} SET BeginTime = NOW() WHERE BeginId = ${sourceIdBegin} AND SourceTableName=#{sourceTableName} AND TargetTableName=#{targetTableName}")
- public int updateTaskBeginTime(
- @Param("migrationTask") String migrationTask,
- @Param("sourceIdBegin") int sourceIdBegin,
- @Param("sourceTableName") String sourceTableName,
- @Param("targetTableName") String targetTableName);
-
- // 获取帖子分表中相应数据 两边都有等于号
- @Select("Select ${selectItems} from ${sourceTable} "
- + "where id >=${idBegin} and id <= ${idEnd} ${andWhere}")
- public List getPostGatherData(
- @Param("sourceTable") String sourceTableName,
- @Param("selectItems") String selectItems,
- @Param("idBegin") int idBegin, @Param("idEnd") int idEnd,
- @Param("andWhere") String andWhere);
-
- // 获取项目分表中相应数据 两边都有等于号
- @Select("Select ${selectItems} from ${sourceTable} "
- + "where id >=${idBegin} and id <= ${idEnd} ${andWhere}")
- public List getPrjGatherData(
- @Param("sourceTable") String sourceTableName,
- @Param("selectItems") String selectItems,
- @Param("idBegin") int idBegin, @Param("idEnd") int idEnd,
- @Param("andWhere") String andWhere);
-
- // 获取项目分表中相应数据 两边都有等于号
- @Select("Select ${selectItems} from ${sourceTable} "
- + "where id >=${idBegin} and id <= ${idEnd} and fork=0 ${andWhere}")
- public List getPrjGatherDataForGit(
- @Param("sourceTable") String sourceTableName,
- @Param("selectItems") String selectItems,
- @Param("idBegin") int idBegin, @Param("idEnd") int idEnd,
- @Param("andWhere") String andWhere);
-
- // 获取job_requirement中相应数据 两边都有等于号
- @Select("Select ${selectItems} from ${sourceTable} "
- + "where id >=${idBegin} and id <= ${idEnd} ${andWhere}")
- public List getJobGatherData(
- @Param("sourceTable") String sourceTableName,
- @Param("selectItems") String selectItems,
- @Param("idBegin") int idBegin, @Param("idEnd") int idEnd,
- @Param("andWhere") String andWhere);
-
- // 更新任务结束时间
- @Update("UPDATE ${migrationTask} SET IsDone = 1,EndTime = NOW(),flow=${moveNum} WHERE BeginId = ${endIdBegin} AND SourceTableName=#{sourceTableName} AND TargetTableName=#{targetTableName}")
- public int updateTaskEndTime(@Param("migrationTask") String migrationTask,
- @Param("endIdBegin") int endIdBegin, @Param("moveNum") int moveNum,
- @Param("sourceTableName") String sourceTableName,
- @Param("targetTableName") String targetTableName);
-
- // 批量获得JobRequirement
- @Select("SELECT * from job_requirements WHERE id > #{id} limit #{batchSize}")
- public List getJobRequirementList(@Param("id") int id,
- @Param("batchSize") int batchSize);
-
-}
\ No newline at end of file
diff --git a/gather_program/src/main/java/org/ossean/gather/sourceDao/PKControlPostsDao.java b/gather_program/src/main/java/org/ossean/gather/sourceDao/PKControlPostsDao.java
deleted file mode 100644
index 5c77d2a59..000000000
--- a/gather_program/src/main/java/org/ossean/gather/sourceDao/PKControlPostsDao.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.ossean.gather.sourceDao;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.ossean.gather.model.PKControlPosts;
-
-
-public interface PKControlPostsDao {
-
- //向表中插入一条数据
- @Insert("insert into ${table} (`url_md5`) values (#{urlMD5})")
- public void insertOneItem(@Param("table") String table, @Param("urlMD5") String urlMD5);
-
- //读取项目urlMD5码对应的数据信息
- @Select("select * from ${table} where url_md5=#{urlMD5}")
- public PKControlPosts selectItemByUrlMD5(@Param("table") String table, @Param("urlMD5") String urlMD5);
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/sourceDao/PKControlProjectsDao.java b/gather_program/src/main/java/org/ossean/gather/sourceDao/PKControlProjectsDao.java
deleted file mode 100644
index 259cf16f9..000000000
--- a/gather_program/src/main/java/org/ossean/gather/sourceDao/PKControlProjectsDao.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.ossean.gather.sourceDao;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.ossean.gather.model.PKControlProjects;
-
-
-public interface PKControlProjectsDao {
-
- //向表中插入一条数据
- @Insert("insert into ${table} (`url_md5`) values (#{urlMD5})")
- public void insertOneItem(@Param("table") String table, @Param("urlMD5") String urlMD5);
-
- //读取项目urlMD5码对应的数据信息
- @Select("select * from ${table} where url_md5=#{urlMD5}")
- public PKControlProjects selectItemByUrlMD5(@Param("table") String table, @Param("urlMD5") String urlMD5);
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/targetDao/PointerDao.java b/gather_program/src/main/java/org/ossean/gather/targetDao/PointerDao.java
deleted file mode 100644
index 613879cab..000000000
--- a/gather_program/src/main/java/org/ossean/gather/targetDao/PointerDao.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package org.ossean.gather.targetDao;
-
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-
-public interface PointerDao {
-
-// // 读取表中最小id值
-// @Select("select min(id) from ${table}")
-// public int getMinId(@Param("table") String table);
-//
-// // 读取表中最大id值
-// @Select("select max(id) from ${table}")
-// public int getMaxId(@Param("table") String table);
-
- @Select("select Pointer from ${table} where SourceTableName=#{SourceTableName} and TargetTableName=#{TargetTableName}")
- public int getPointer(@Param("table") String table,
- @Param("SourceTableName") String SourceTableName,
- @Param("TargetTableName") String TargetTableName);
-
- @Update("update ${table} set Pointer=#{Pointer} where SourceTableName=#{SourceTableName} and TargetTableName=#{TargetTableName}")
- public void updatePointer(@Param("table") String table,
- @Param("SourceTableName") String SourceTableName,
- @Param("TargetTableName") String TargetTableName,
- @Param("Pointer") int Pointer);
-
- @Insert("insert into ${table} (`SourceTableName`,`TargetTableName`,`Pointer`) values (#{SourceTableName},#{TargetTableName},#{Pointer})")
- public void insertPointer(@Param("table") String table,
- @Param("SourceTableName") String SourceTableName,
- @Param("TargetTableName") String TargetTableName,
- @Param("Pointer") int Pointer);
-
-}
diff --git a/gather_program/src/main/java/org/ossean/gather/targetDao/TargetDao.java b/gather_program/src/main/java/org/ossean/gather/targetDao/TargetDao.java
deleted file mode 100644
index 7f9452eb4..000000000
--- a/gather_program/src/main/java/org/ossean/gather/targetDao/TargetDao.java
+++ /dev/null
@@ -1,144 +0,0 @@
-package org.ossean.gather.targetDao;
-
-import java.util.Set;
-
-import org.apache.ibatis.annotations.Delete;
-import org.apache.ibatis.annotations.Insert;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Select;
-import org.apache.ibatis.annotations.Update;
-import org.ossean.gather.model.GatherProject;
-import org.ossean.gather.model.JobRequirement;
-import org.ossean.gather.model.RelativeMemo;
-import org.ossean.gather.model.Taggings;
-
-public interface TargetDao {
-
- // 获得relative_memos中已存在的url_md5集合
- @Select("SELECT url_md5 AS md5 FROM ${targetTable}")
- public Set