gather_program/sql/open_source_projects.sql.tar.gz

ready for crawler

.gitignore

@@ -72,7 +72,6 @@
 /trustie2/files/*
 /trustie2/log/*
 /trustie2/tmp/*
-/trustie2/public/images/avatars/*
 /trustie2/Gemfile.lock
 /trustie2/db/schema.rb
 !/trustie2/solr/conf
@@ -109,4 +108,12 @@
 /gather_program/target/
 /gather_program/log/*
 /gather_program/.settings/
-/gather_program/.classpath
+/gather_program/.classpath
+
+
+/crawler/dailyScheduledCrawler/fetch_networks/target/
+/crawler/dailyScheduledCrawler/fetch_networks/log/*
+/crawler/daily_scheduler/log/*
+/crawler/daily_scheduler/log_mem/*
+/crawler/moreSmarterCrawler/fetch_networks/target/
+/crawler/moreSmarterCrawler/fetch_networks/log/*

crawler/dailyScheduledCrawler/fetch_networks/.classpath

@@ -1,25 +1,33 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
+	<classpathentry exported="true" kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="src" output="target/classes" path="src/main/java">
 		<attributes>
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
+	<classpathentry including="**/*.java" kind="src" output="target/classes" path="src/main/resources"/>
 	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
 		<attributes>
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry including="**/*.java" kind="src" path="src/main/resources"/>
 	<classpathentry kind="src" path="src/main/assembly"/>
-	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+	<classpathentry kind="src" output="target/classes" path="sites">
 		<attributes>
+			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
+	<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/junit-4.11"/>
+	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>

crawler/dailyScheduledCrawler/fetch_networks/.project

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>fetchnetworks</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.maven.ide.eclipse.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.maven.ide.eclipse.maven2Nature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
+	</natures>
+</projectDescription>

crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs

@@ -0,0 +1,5 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding/<project>=UTF-8

crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs

@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+line.separator=\n

crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs

@@ -0,0 +1,14 @@
+#Sun Jan 04 15:44:05 CST 2015
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.7
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.source=1.7

crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs

@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1

crawler/dailyScheduledCrawler/fetch_networks/bin/51cto_blog.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='51cto_blog'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/apache.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='apache'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/bytes.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='bytes'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_news.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='cnblogs_news'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_solved.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='cnblogs_q_solved'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='cnblogs_q_unsolved'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/code_project.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='code_project'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_ask.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='csdn_ask'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_blog.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='csdn_blog'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_topic.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='csdn_topic'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx256m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/dewen_question.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='dewen_question'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/freecode.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='freecode'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/gna.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='gna'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_post.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='ibm_post'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_project.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='ibm_project'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_ask.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='iteye_ask'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_blog.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='iteye_blog'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/lagou.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='lagou'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/linuxtone.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='linuxtone'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/lupaworld.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='lupaworld'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/neitui.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='neitui'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/openhub.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='openhub'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_project.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='oschina_project'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_question.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='oschina_question'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/phpchina.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='phpchina'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/resources/log4j.xml

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+	<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
+		<layout class="org.apache.log4j.PatternLayout">
+			<param name="threshold" value="ERROR" />
+			<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
+		</layout>
+	</appender>
+
+	<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
+		<param name="File" value="./log/error.log" />
+		<param name="threshold" value="ERROR" />
+		<layout class="org.apache.log4j.PatternLayout">
+			<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
+		</layout>
+	</appender>
+
+	<appender name="file_log" class="org.apache.log4j.DailyRollingFileAppender">
+		<param name="File" value="./log/webmagic.log" />
+		<layout class="org.apache.log4j.PatternLayout">
+			<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
+		</layout>
+	</appender>
+
+
+	<!-- 邮件（只有ERROR时才会发送！） -->
+	<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
+		<param name="threshold" value="debug" />
+		<!-- 日志的错误级别 <param name="threshold" value="fatal"/> -->
+		<!-- 缓存文件大小，日志达到512K时发送Email -->
+		<param name="BufferSize" value="1" />
+		<!-- 单位K -->
+		<param name="From" value="ossean_debug@163.com" />
+		<param name="SMTPHost" value="smtp.163.com" />
+		<param name="Subject" value="ossean-crawler-debug-log4jMessage" />
+		<param name="To" value="gcm3651@126.com" />
+		<param name="SMTPUsername" value="ossean_debug" />
+		<param name="SMTPPassword" value="goodwell123" />
+		<layout class="org.apache.log4j.PatternLayout">
+			<param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss.SSS} [%p]-[%c] %m%n" />
+		</layout>
+	</appender>
+
+	<!-- 数据库状态 -->
+	<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
+		<param name="URL"
+			value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8" />
+		<param name="driver" value="com.mysql.jdbc.Driver" />
+		<param name="user" value="root" />
+		<param name="password" value="1234" />
+		<param name="sql"
+			value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%p','%c','%m','%l')" />
+		<!-- <layout class="org.apache.log4j.PatternLayout"> <param name="ConversionPattern" 
+			value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES 
+			('%d{yyyy-MM-dd HH:mm:ss}','%t','%.50p','%.50c','%.1000m','%.50l')" /> </layout> -->
+		<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都 FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
+		<filter class="org.apache.log4j.varia.LevelRangeFilter">
+			<param name="LevelMin" value="DEBUG" />
+			<param name="LevelMax" value="FATAL" />
+		</filter>
+	</appender>
+
+	<logger name="org.apache" additivity="false">
+		<level value="warn" />
+		<appender-ref ref="stdout" />
+	</logger>
+
+	<root>
+		<level value="info" />
+		<appender-ref ref="stdout" />
+		<!-- <appender-ref ref="file" /> -->
+		<!-- <appender-ref ref="file_log" /> -->
+		<!-- <appender-ref ref="MAIL" /> -->
+		<!-- <appender-ref ref="DATABASE" /> -->
+	</root>
+
+</log4j:configuration>

crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml

@@ -9,15 +9,24 @@
     </bean>
 
     <bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
-        <property name="basePackage" value="net.trustie.webmagic.dao" />
+        <property name="basePackage" value="net.trustie.webmagic" />
     </bean>
 
     <bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
           destroy-method="close">
         <property name="driverClassName" value="com.mysql.jdbc.Driver" />
-        <property name="url" value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8" />
-        <property name="username" value="root" />
-        <property name="password" value="root" />
+        <property name="url" value="jdbc:mysql://192.168.80.104:3306/pages?characterEncoding=UTF-8" />
+        <property name="username" value="influx" />
+        <property name="password" value="influx1234" />
+        <property name="timeBetweenEvictionRunsMillis">
+        		   <value>21600000</value><!--6 hours-->
+		  </property>
+		  		
+		  <property name="minEvictableIdleTimeMillis">
+		  		   <value>21600000</value><!--connection的空闲时间大于这个值,就直接被关闭,并从连接池中删除-->
+		  </property>
     </bean>
 
-</beans>
+
+
+</beans>

crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext.xml

@@ -12,5 +12,5 @@
     <context:annotation-config/>
 
     <context:component-scan base-package="net.trustie.webmagic"/>
-	<!--context:component-scan base-package="net.trustie.webmagic.dao"/-->
+
 </beans>

crawler/dailyScheduledCrawler/fetch_networks/bin/resources/testDBlogger.xml

@@ -0,0 +1,43 @@
+<!-- 输出通道"DATABASE"，输出方式是：输出所有级别的LOG到数据库 -->
+<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
+    <!--数据库的驱动，这里用的是MSSQL-->
+    <param name="driver" value="com.microsoft.jdbc.sqlserver.SQLServerDriver"/>
+    <!--这是Oracle用的驱动，在这里用MSSQL所以屏蔽掉了
+         <param name="driver" value="oracle.jdbc.driver.OracleDriver" />
+    -->
+    <!--要连接的数据库-->
+    <param name="URL" value="jdbc:microsoft:sqlserver://192.168.0.120:
+
+                  1433;DatabaseName=test" />
+    <!--连接数据库的用户名-->
+    <param name="user" value="sa" />
+    <!--连接数据库的密码-->
+    <param name="password" value="sa" />
+    <!--向MSSQL数据库表LOG中插入数据的sql语句-->
+    <param name="sql" value=" INSERT INTO LOG
+
+           (LOGDATE,LOGLEVEL,LOGCLASS,LOGLOGGER,LOGMESSAGE)
+
+           values ('%d{yyyy-MM-dd HH:mm:ss}','%.50p', '%.50c',
+
+                   '%.50l', '%.1000m')"/>
+
+
+
+    <!-- Oracle 的 insert 语句
+            <param name="sql" value=" insert into logrecord
+
+           (id,packageid,userid,syscodeid,info,logtime,loglevel)
+
+           values(?,?,?,?,?,to_date('%d{yyyy-MM-dd HH:mm:ss}',
+
+           'yyyy-MM-dd HH24:mi:ss'),'%p')" />
+    -->
+    <!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都
+
+           FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
+    <filter class="org.apache.log4j.varia.LevelRangeFilter">
+        <param name="LevelMin" value="ERROR"/>
+        <param name="LevelMax" value="FATAL"/>
+    </filter>
+</appender>

crawler/dailyScheduledCrawler/fetch_networks/bin/sample_start.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='freecode'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/slashdot.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='slashdot'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/softpedia.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='softpedia'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/sourceforge.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+SITE='sourceforge'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/stackoverflow.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
+SITE='stackoverflow'
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+find ./target/classes/spring |xargs rm -f -r
+find ./target/classes/sites |xargs rm -f -r
+
+tmp='./bin/resources'
+tmp='./sites':$tmp
+tmp='./target/classes':$tmp
+tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn48m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#-XX:MinPermSize=128m"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
+java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
+#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

crawler/dailyScheduledCrawler/fetch_networks/bin/start_all.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+sh bin/cnblogs_news.sh
+sh bin/cnblogs_q_solved.sh
+sh bin/cnblogs_q_unsolved.sh
+sh bin/csdn_ask.sh
+sh bin/csdn_blog.sh
+sh bin/csdn_topic.sh
+sh bin/dewen_question.sh
+sh bin/iteye_ask.sh
+sh bin/openhub.sh
+sh bin/oschina_project.sh
+sh bin/oschina_question.sh
+sh bin/sourceforge.sh
+sh bin/stackoverflow.sh

crawler/dailyScheduledCrawler/fetch_networks/ossean-bak.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+<comment>Spider</comment>
+<entry key="pageF">1</entry>
+<entry key="domain">one</entry>
+<entry key="threadNum">5</entry>
+<entry key="startPageIndex">1</entry>
+<entry key="isSpawnUrl">true</entry>
+<entry key="endInPageIndex">20000</entry>
+<entry key="retryTimes">5</entry>
+<entry key="timeOut">30000</entry>
+<entry key="sleepTimeThread">5000</entry>
+<entry key="cycleRetryTimes">5</entry>
+<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+<entry key="sleepTimeSpider">10000</entry>
+<entry key="startUrl">http://www.oschina.net/question/</entry>
+<entry key="urlList">http://www\.oschina\.net/question\?catalog=[12]\&show=active\&p=\d+</entry>
+<entry key="urlPost">http://www\.oschina\.net/question/\d+_\d+</entry>
+<entry key="offset">0</entry>
+<entry key="limitLine">10</entry>
+<entry key="noUrlWaitTime">10000</entry>
+<entry key="postTableName">html_test</entry>
+<entry key="listTableName">target_url</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/pom.xml

@@ -1,10 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
 	<modelVersion>4.0.0</modelVersion>
 
 	<groupId>net.trustie</groupId>
-	<artifactId>tFlow</artifactId>
+	<artifactId>fetchnetworks</artifactId>
 	<version>1.0-SNAPSHOT</version>
 
 	<properties>
@@ -17,6 +18,7 @@
 
 	<build>
 		<sourceDirectory>${basedir}/src/main/java</sourceDirectory>
+
 		<plugins>
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
@@ -37,22 +39,90 @@
 			</plugin>
 		</plugins>
 	</build>
+
 	<dependencies>
+		<dependency>
+			<groupId>xerces</groupId>
+			<artifactId>xerces</artifactId>
+			<version>2.4.0</version>
+		</dependency>
+		<dependency>
+			<groupId>dom4j</groupId>
+			<artifactId>dom4j</artifactId>
+			<version>1.6.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.json</groupId>
+			<artifactId>json</artifactId>
+			<version>20141113</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>14.0</version>
+		</dependency>
+
+		<dependency>
+			<groupId>us.codecraft</groupId>
+			<artifactId>xsoup</artifactId>
+			<version>0.2.4</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.github.dreamhead</groupId>
+			<artifactId>moco-core</artifactId>
+			<version>0.10.0</version>
+		</dependency>
+
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-log4j12</artifactId>
 			<version>1.7.7</version>
 		</dependency>
+
+
+
 		<dependency>
 			<groupId>commons-collections</groupId>
 			<artifactId>commons-collections</artifactId>
 			<version>3.2.1</version>
 		</dependency>
+
+		<dependency>
+			<groupId>org.assertj</groupId>
+			<artifactId>assertj-core</artifactId>
+			<version>1.7.0</version>
+		</dependency>
+
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-io</artifactId>
 			<version>1.3.2</version>
 		</dependency>
+
+		<dependency>
+			<groupId>com.jayway.jsonpath</groupId>
+			<artifactId>json-path</artifactId>
+			<version>0.8.1</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.alibaba</groupId>
+			<artifactId>fastjson</artifactId>
+			<version>1.2.3</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.jsoup</groupId>
+			<artifactId>jsoup</artifactId>
+			<version>1.7.2</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.httpcomponents</groupId>
+			<artifactId>httpclient</artifactId>
+			<version>4.3.4</version>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework</groupId>
 			<artifactId>spring-jdbc</artifactId>
@@ -63,6 +133,11 @@
 			<artifactId>commons-lang3</artifactId>
 			<version>3.1</version>
 		</dependency>
+		<dependency>
+			<groupId>javax.servlet</groupId>
+			<artifactId>servlet-api</artifactId>
+			<version>2.5</version>
+		</dependency>
 		<dependency>
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>
@@ -79,21 +154,49 @@
 			<version>4.7</version>
 			<scope>test</scope>
 		</dependency>
+
 		<dependency>
 			<groupId>org.mybatis</groupId>
 			<artifactId>mybatis</artifactId>
 			<version>3.1.1</version>
 		</dependency>
+
 		<dependency>
 			<groupId>org.mybatis</groupId>
 			<artifactId>mybatis-spring</artifactId>
 			<version>1.1.1</version>
 		</dependency>
+
 		<dependency>
 			<groupId>org.springframework</groupId>
 			<artifactId>spring-test</artifactId>
 			<version>${spring-version}</version>
 			<scope>test</scope>
 		</dependency>
+
+		<dependency>
+			<groupId>net.trustie</groupId>
+			<artifactId>webmagic-core-fix</artifactId>
+			<version>1.0</version>
+			<scope>system</scope>
+			<systemPath>${project.basedir}/lib/webmagic-core-fix.jar</systemPath>
+		</dependency>
+
+		<dependency>
+			<groupId>net.trustie</groupId>
+			<artifactId>webmagic-extension-fix</artifactId>
+			<version>1.0</version>
+			<scope>system</scope>
+			<systemPath>${project.basedir}/lib/webmagic-extension-fix.jar</systemPath>
+		</dependency>
+
+		<dependency>
+			<groupId>org.seleniumhq.selenium</groupId>
+			<artifactId>selenium-java</artifactId>
+			<version>2.42.2</version>
+		</dependency>
+
+
 	</dependencies>
+
 </project>

crawler/dailyScheduledCrawler/fetch_networks/sites/51cto_blog.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>51cto</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>div.rightbox>div.r_li>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://blog.51cto.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://blog.51cto.com/original/0/</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>16</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>51cto_blog</domain>
+	    <minInterval>20000</minInterval>
+	    <maxInterval>30000</maxInterval>
+	    <minlongSleepTime>600000</minlongSleepTime>
+	    <maxlongSleepTime>1200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/Copy of sample_site.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">csdn_topic</entry>
+    <!-- 有序爬取中，起始页页码-->
+    <entry key="startPageIndex">1</entry>
+    <!-- 有序爬取中，结束页页码-->
+    <entry key="endInPageIndex">10</entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
+    <!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
+    <entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry> 
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">50</entry>
+    <!-- 列表页更新完后休息时间	单位 为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">20000</entry>
+    <!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry> 
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+    
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/apache.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+<site>
+	<subSites>
+	    	<subSite>
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>1</endInPageIndex>
+	    		<urlPost>table>tbody>tr>td.body>div>div>ul>li> a</urlPost>
+	    		<fixAllRelativeHrefs>http://projects.apache.org</fixAllRelativeHrefs>
+	    		<prefixUrl>http://projects.apache.org/indexes/alpha.html</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>864000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>apache</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>30000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">51cto_blog</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">30</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://blog.51cto.com/original/0/1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">div.rightbox>div.r_li>h4>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://blog.51cto.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://blog.51cto.com/original/0/</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">16</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of apache.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">apache</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">30</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://projects.apache.org/indexes/alpha.html</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">table>tbody>tr>td.body>div>div>ul>li> a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://projects.apache.org/indexes/alpha.html</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://projects.apache.org/indexes/alpha.html</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">1</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">8640000</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">cnblogs_news</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">4845</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://news.cnblogs.com/n/page/1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div#news_list>div.news_block>div.content>h2.news_entry>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://news.cnblogs.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://news.cnblogs.com/n/page/</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">5</entry>
+
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">cnblogs_q_solved</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">160</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">4000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://q.cnblogs.com/list/solved?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://q.cnblogs.com/list/solved?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">2</entry>
+
+    
+     <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">cnblogs_q_unsolved</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">160</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">4000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://q.cnblogs.com/list/unsolved?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://q.cnblogs.com/list/unsolved?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">2</entry>
+    
+    
+     <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of code_project.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">codeproject</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">21</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">div.content-list-item>div.title>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://www.codeproject.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">1</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">dewen_question</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">1114</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://www.dewen.io/questions?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://www.dewen.io</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://www.dewen.io/questions?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">1</entry>
+    
+    
+     <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">432000</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of gna.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">gna</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">1348</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=1&amp;max_rows=1#results</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">table>tbody>tr>td>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://gna.org</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">&amp;max_rows=1#results</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">200</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">432000</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">lupaworld</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">2</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">479</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://www.lupaworld.com/forum-13746-2.html</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">table#threadlisttableid>tbody>tr>th>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://www.lupaworld.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://www.lupaworld.com/forum-13746-</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">.html</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">2</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">2592000</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of openhub.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!--  -->
+    <entry key="domain">openhub</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">66784</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">60000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">https://www.openhub.net/p?page=1&amp;q=&amp;sort=users</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.project>h2.title>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">60000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">https://www.openhub.net</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">https://www.openhub.net/p?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">&amp;q=&amp;sort=users</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">500</entry>
+    <!-- 列表页更新完后休息时间	为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">100000</entry>
+    
+
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of phpchina.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">phpchina</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">1289</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://bbs.phpchina.com/forum-17-1.html</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">table#threadlisttableid>tbody>tr>th>a.s.xst</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://bbs.phpchina.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://bbs.phpchina.com/forum-17-</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">.html</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">2</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">1296000</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of softpedia.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">softpedia</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">30</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://linux.softpedia.com/index1.shtml</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">div#sjmp>div>h4>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://linux.softpedia.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://linux.softpedia.com/index</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">.shtml</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">10</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!--  -->
+    <entry key="domain">sourceforge</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">17621</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">60000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">10000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://sourceforge.net/directory/?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">ul.projects>li>div.project_info>header>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://sourceforge.net</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://sourceforge.net/directory/?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">500</entry>
+    <!-- 列表页更新完后休息时间	为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">250000</entry>
+    
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!-- 有序爬取时，每次生成页面链接数目-->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">stackoverflow</entry>
+    <!-- 线程数目-->
+    <entry key="threadNum">1</entry>
+    <!-- 有序爬取中，起始页页码-->
+    <entry key="startPageIndex">1</entry>
+    <!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
+    <entry key="isSpawnUrl">true</entry>
+    <!-- 有序爬取中，结束页页码-->
+    <entry key="endInPageIndex">25000</entry>
+    <!-- 页面下载超时，重试次数-->
+    <entry key="retryTimes">5</entry>
+    <!-- 下载时，连接超时等待时间-->
+    <entry key="timeOut">60000</entry>
+    <!-- 超时时间，单位是毫秒-->
+    <entry key="sleepTimeThread">0</entry>
+    <!-- 循环重试次数-->
+    <entry key="cycleRetryTimes">5</entry>
+    <!-- 请求头的UserAgent信息-->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
+    <entry key="sleepTimeSpider">10000</entry>
+    <!-- 起始种子URL -->
+    <entry key="startUrl">http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div#mainbar>div#questions>div.question-summary>div.summary>h3>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
+    <entry key="fixAllRelativeHrefs">http://stackoverflow.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">250</entry>
+    <!-- 列表页更新完后休息时间	单位 为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">20000</entry>
+    
+    <entry key="minInterval">2000</entry>
+    <entry key="maxInterval">3000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">2400000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">7200000</entry>
+    <entry key="MaxlongSleepInterval">14400000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">linuxtone</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">64</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://bbs.linuxtone.org/forum-15-1.html</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">table>tbody>tr>th>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://bbs.linuxtone.org</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://bbs.linuxtone.org/forum-15-</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">.html</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">2</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">2592000</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">iteye_blog</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">15</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://www.iteye.com/blogs?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">div.content>h3>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://www.iteye.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://www.iteye.com/blogs?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">2</entry>
+  
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/bytes.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name>question</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>2000</endInPageIndex>
+	    			    			    			    		<urlPost>table#threadslist>tbody>tr>td>div>div.threadbit_title>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bytes.com</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bytes.com/answers/</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl>/</postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>2</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>bytes</domain>
+		    	    <minInterval>10000</minInterval>
+		    	    	    <maxInterval>20000</maxInterval>
+		    	    	    	    <minlongSleepTime>1800000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>7200000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>3600000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_news.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>4845</endInPageIndex>
+	    		<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
+	    		<fixAllRelativeHrefs>http://news.cnblogs.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://news.cnblogs.com/n/page/</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>5</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>cnblogs_news</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>
+

crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_solved.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>160</endInPageIndex>
+	    		<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
+	    		<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://q.cnblogs.com/list/solved?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>cnblogs_q_solved</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<site>
+	<subSites>
+	    	<subSite>
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
+	    		<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://q.cnblogs.com/list/unsolved?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>cnblogs_q_unsolved</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/code_project.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>259</endInPageIndex>
+	    		<urlPost>div.content-list-item>div.title>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.codeproject.com</fixAllRelativeHrefs>
+	    		<!--<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</prefixUrl>-->
+			<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=366&amp;la_app=20&amp;la_minscore=1.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>codeproject</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_ask.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>2496</endInPageIndex>
+	    			    			    			    		<urlPost>div.common_con>div.questions_detail_con>dl>dt>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://ask.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://ask.csdn.net/?page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>25</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>csdn_ask</domain>
+		    	    <minInterval>30000</minInterval>
+		    	    	    <maxInterval>60000</maxInterval>
+		    	    	    	    <minlongSleepTime>7200000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>10800000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>1800000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_blog.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>101</endInPageIndex>
+	    			    			    			    		<urlPost>div.main_center>div.blog_list>h1>a:not(.category)</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://blog.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://blog.csdn.net/index.html?&amp;page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>3</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>csdn_blog</domain>
+		    	    <minInterval>30000</minInterval>
+		    	    	    <maxInterval>60000</maxInterval>
+		    	    	    	    <minlongSleepTime>7200000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>10800000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>1800000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_topic.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name>mobile</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>808</endInPageIndex>
+	    			    			    			    		<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.csdn.net/forums/Mobile?page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>8</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    	<subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    		    		<!-- 普通网站name不用写 -->
+	    		<name>CloudComputing</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>22</endInPageIndex>
+	    			    			    			    		<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.csdn.net/forums/CloudComputing?page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    	<subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    		    		<!-- 普通网站name不用写 -->
+	    		<name>java</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>2555</endInPageIndex>
+	    			    			    			    		<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.csdn.net/forums/Java?page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>4</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    	<subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    		    		<!-- 普通网站name不用写 -->
+	    		<name>web</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>1853</endInPageIndex>
+	    			    			    			    		<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.csdn.net/forums/WebDevelop?page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>3</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    	<subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    		    		<!-- 普通网站name不用写 -->
+	    		<name>OtherLanguage</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>336</endInPageIndex>
+	    			    			    			    		<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.csdn.net/forums/OtherLanguage?page=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>csdn_topic</domain>
+		    	    <minInterval>30000</minInterval>
+		    	    	    <maxInterval>60000</maxInterval>
+		    	    	    	    <minlongSleepTime>7200000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>10800000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>1800000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/dewen_question.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>1114</endInPageIndex>
+	    		<urlPost>div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.dewen.io</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.dewen.io/questions?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>432000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>dewen_question</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/freecode.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">5</entry>
+    <!--  -->
+    <entry key="domain">freecode</entry>
+    <!--  -->
+    <entry key="threadNum">5</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">4799</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">30000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">10000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://www.freecode.com/projects?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.project>h2.release-head>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">60000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://www.freecode.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://www.freecode.com/projects?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">5</entry>
+    <!-- 列表页更新完后休息时间	为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">20000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/gna.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>1348</endInPageIndex>
+	    		<urlPost>table>tbody>tr>td>a</urlPost>
+	    		<fixAllRelativeHrefs>http://gna.org</fixAllRelativeHrefs>
+	    		<prefixUrl>http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=</prefixUrl>
+	   			<postfixUrl>&amp;max_rows=1#results</postfixUrl>
+	   			<incrementalPages>100</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>gna</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_post.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name>blog</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>138</endInPageIndex>
+	    			    			    			    		<urlPost>table>tbody>tr>td>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://www.ibm.com/developerworks/cn/views/opensource/libraryview.jsp?site_id=10&amp;contentarea_by=%E6%89%80%E6%9C%89%E4%B8%93%E5%8C%BA&amp;sort_by=&amp;sort_order=2&amp;start=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl>00&amp;end=13874&amp;topic_by=-1&amp;product_by=&amp;type_by=%E6%89%80%E6%9C%89%E7%B1%BB%E5%88%AB&amp;show_abstract=false&amp;search_by=&amp;industry_by=&amp;series_title_by=</postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>6048000</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>ibm_post</domain>
+		    	    <minInterval>10000</minInterval>
+		    	    	    <maxInterval>20000</maxInterval>
+		    	    	    	    <minlongSleepTime>1800000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>7200000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>3600000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_project.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>1</endInPageIndex>
+	    			    			    			    		<urlPost>table>tbody>tr>td>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://www.ibm.com/developerworks/views/opensource/projects.jsp</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>864000</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>ibm_project</domain>
+		    	    <minInterval>10000</minInterval>
+		    	    	    <maxInterval>20000</maxInterval>
+		    	    	    	    <minlongSleepTime>1800000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>7200000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>3600000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_ask.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">5</entry>
+    <!--  -->
+    <entry key="domain">iteye_ask</entry>
+    <!--  -->
+    <entry key="threadNum">5</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">5329</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">30000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://www.iteye.com/ask?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div#ask_list>div.question-summary>div.summary>h3>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">60000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://www.iteye.com/</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://www.iteye.com/ask?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">50</entry>
+    <!-- 列表页更新完后休息时间	为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">20000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_bbs.xml

@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<name>mobile</name>
+	    		<startPageIndex>z</startPageIndex>
+	    		<endInPageIndex>111</endInPageIndex>
+	    		<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.iteye.com/forums/board/mobile?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>16</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	<subSite>
+	    		<name>language</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>242</endInPageIndex>
+	    		<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.iteye.com/forums/board/language?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	<subSite>
+	    		<name>web</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>320</endInPageIndex>
+	    		<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.iteye.com/forums/board/web?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<name>java</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>754</endInPageIndex>
+	    		<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.iteye.com/forums/board/Java?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<name>tech</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>242</endInPageIndex>
+	    		<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.iteye.com/forums/board/Tech?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>iteye_bbs</domain>
+	    <minInterval>30000</minInterval>
+	    <maxInterval>50000</maxInterval>
+	    <minlongSleepTime>3600000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>1800000</minlongSleepInterval>
+	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_blog.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<name>blog</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>15</endInPageIndex>
+	    		<urlPost>div.content>h3>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.iteye.com/blogs?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>iteye_blog</domain>
+	    <minInterval>30000</minInterval>
+	    <maxInterval>50000</maxInterval>
+	    <minlongSleepTime>3600000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>1800000</minlongSleepInterval>
+	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/javaforge.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">1</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">javaforge</entry>
+    <!--  -->
+    <entry key="threadNum">1</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">31</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">5000</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://javaforge.com/listProjects.spr?page=1&amp;displayAllProjects=on</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->		
+    <entry key="urlPost">table#entry>tbody>tr>td:nth-child(1) > a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://javaforge.com</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://javaforge.com/listProjects.spr?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl">&amp;displayAllProjects=on</entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">10</entry>
+    
+    <!-- 列表页更新完后休息时间	单位为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">86400</entry>
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/lagou.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name>houduan</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>30</endInPageIndex>
+	    			    			    			    		<urlPost> #container > div.content>ul> li> div> div > a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://www.lagou.com</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://www.lagou.com/jobs/list_%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91?kd=%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91&amp;spc=1&amp;pl=&amp;gj=&amp;xl=&amp;yx=&amp;gx=&amp;st=&amp;labelWords=label&amp;lc=&amp;workAddress=&amp;city=%E5%85%A8%E5%9B%BD&amp;requestId=&amp;pn=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>16</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>lagou</domain>
+		    	    <minInterval>10000</minInterval>
+		    	    	    <maxInterval>20000</maxInterval>
+		    	    	    	    <minlongSleepTime>1800000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>7200000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>3600000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/linuxtone.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>System</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>64</endInPageIndex>
+	    		<urlPost>table>tbody>tr>th>a</urlPost>
+	    		<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
+	    		<prefixUrl>http://bbs.linuxtone.org/forum-15-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>2592000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>server</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>table>tbody>tr>th>a</urlPost>
+	    		<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
+	    		<prefixUrl>http://bbs.linuxtone.org/forum-22-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>2592000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		    		<!-- 普通网站name不用写 -->
+	    		<name>dbserver</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>32</endInPageIndex>
+	    			    			    			    		<urlPost>table>tbody>tr>th>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.linuxtone.org/forum-24-</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl>.html</postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>2592000</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    	<subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    		    		<!-- 普通网站name不用写 -->
+	    		<name>yngz</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>32</endInPageIndex>
+	    			    			    			    		<urlPost>table>tbody>tr>th>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.linuxtone.org/forum-49-</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl>.html</postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>2592000</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    	<subSite>
+	    			    			    			    			    			    			   				   				    			    			    		    		    		<!-- 普通网站name不用写 -->
+	    		<name>fzjh</name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>24</endInPageIndex>
+	    			    			    			    		<urlPost>table>tbody>tr>th>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://bbs.linuxtone.org/forum-26-</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl>.html</postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>2592000</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>linuxtone</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/lupaworld.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>479</endInPageIndex>
+	    		<urlPost>table#threadlisttableid>tbody>tr>th>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.lupaworld.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.lupaworld.com/forum-13746-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>2592000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>lupaworld</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/neitui.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>610</endInPageIndex>
+	    		<urlPost> ul>li>div.cont>div.jobnote.clearfix > div.jobnote-l>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.neitui.me</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.neitui.me/neitui/type=all&amp;page=</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>16</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>neitui</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/open_open.xml

@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>dev</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>370</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/1?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>language</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>652</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/36?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>website</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>41</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/57?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>server</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>145</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/75?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>database</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>102</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/90?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>plugin</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>53</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/52?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>OS</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>42</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/155?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	
+	    	
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>manage</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>12</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/319?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>dev_manage</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>9</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/159?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>util</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>7</endInPageIndex>
+	    		<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
+	    		<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://www.open-open.com/lib/list/322?pn=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>open_open</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/openhub.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>66784</endInPageIndex>
+	    		<urlPost>div.well>h2.title>a</urlPost>
+	    		<fixAllRelativeHrefs>https://www.openhub.net</fixAllRelativeHrefs>
+	    		<!--<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
+	   			<postfixUrl>&amp;q=&amp;sort=users</postfixUrl>
+	   			<incrementalPages>300</incrementalPages>
+	    		<incrementSleepTime>300000</incrementSleepTime> -->
+	    		
+	   		<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
+	   			<postfixUrl>&amp;q=&amp;sort=new</postfixUrl>
+	   			<incrementalPages>8</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	   			
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>openhub</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_project.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>2044</endInPageIndex>
+	    			    			    			    		<urlPost>div.ProjectList>ul.List>li>h3>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://www.oschina.net/project/list?prefix=&amp;sort=time&amp;p=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>oschina_project</domain>
+		    	    <minInterval>30000</minInterval>
+		    	    	    <maxInterval>60000</maxInterval>
+		    	    	    	    <minlongSleepTime>7200000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>10800000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>1800000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_question.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+		    	<subSite>
+		    		    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    			    		<startPageIndex>1</startPageIndex>
+	    			    			    		<endInPageIndex>2361</endInPageIndex>
+	    			    			    			    		<urlPost>ul.list>li.question>div.question-detail>strong>a</urlPost>
+	    			    			    			    			    		<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
+	    			    			    			    			    			    		<prefixUrl>http://www.oschina.net/question?catalog=1&amp;show=active&amp;p=</prefixUrl>
+	    			    			    			    			    			    			   			<postfixUrl></postfixUrl>
+	    			    			    			    			    			    			   				   			<incrementalPages>1</incrementalPages>
+	    			    			    			    			    			    			   				   				    		<incrementSleepTime>86400</incrementSleepTime>
+	    			    			    			    			    			    			   				   				    			    		<mode>0</mode>
+	    			    			    			    			    			    			   				   				    			    			    	</subSite>
+	    			    			    			    			    			    			   				   				    			    			    		 </subSites>
+	    			    			    			    			    			    			   				   				    			    			    		     
+	<properties>
+		    <domain>oschina_question</domain>
+		    	    <minInterval>30000</minInterval>
+		    	    	    <maxInterval>60000</maxInterval>
+		    	    	    	    <minlongSleepTime>7200000</minlongSleepTime>
+		    	    	    	    	    <maxlongSleepTime>10800000</maxlongSleepTime>
+		    	    	    	    	    	    <minlongSleepInterval>1800000</minlongSleepInterval>
+		    	    	    	    	    	    	    <maxlongSleepInterval>3600000</maxlongSleepInterval>
+		    	    	    	    	    	    	    	</properties>
+
+</site>
+
+  

crawler/dailyScheduledCrawler/fetch_networks/sites/ossean.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!--  -->
+    <entry key="pageF">3</entry>
+    <!--  -->
+    <entry key="domain">ossean</entry>
+    <!--  -->
+    <entry key="threadNum">3</entry>
+    <!--  -->
+    <entry key="startPageIndex">1</entry>
+    <!--  -->
+    <entry key="isSpawnUrl">true</entry>
+    <!--  -->
+    <entry key="endInPageIndex">5777</entry>
+    <!--  -->
+    <entry key="retryTimes">5</entry>
+    <!--  -->
+    <entry key="timeOut">10000</entry>
+    <!--  -->
+    <entry key="sleepTimeThread">0</entry>
+    <!--  -->
+    <entry key="cycleRetryTimes">5</entry>
+    <!--  -->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!--  -->
+    <entry key="sleepTimeSpider">500</entry>
+    <!-- 起始页 -->
+    <entry key="startUrl">http://localhost/open_source_projects?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">ul.projects>li.project-table>div.root>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!--  -->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!--  -->
+    <entry key="fixAllRelativeHrefs">http://localhost</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://localhost/open_source_projects?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">5</entry>
+    <!-- 列表页更新完后休息时间	为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">0</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/phpchina.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>ask</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>1289</endInPageIndex>
+	    		<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
+	    		<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://bbs.phpchina.com/forum-17-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>server_os</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>91</endInPageIndex>
+	    		<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
+	    		<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://bbs.phpchina.com/forum-195-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>web</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>81</endInPageIndex>
+	    		<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
+	    		<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://bbs.phpchina.com/forum-213-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>db</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>43</endInPageIndex>
+	    		<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
+	    		<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://bbs.phpchina.com/forum-196-</prefixUrl>
+	   			<postfixUrl>.html</postfixUrl>
+	   			<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>1296000</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>phpchina</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/sample_site.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+    <comment>Spider</comment>
+    <!-- 有序爬取时，每次生成页面链接数目-->
+    <entry key="pageF">3</entry>
+    <!-- 爬虫所爬内容	最好定义与脚本中的SITE名字相同的名字 -->
+    <entry key="domain">csdn_topic</entry>
+    <!-- 线程数目-->
+    <entry key="threadNum">3</entry>
+    <!-- 有序爬取中，起始页页码-->
+    <entry key="startPageIndex">1</entry>
+    <!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
+    <entry key="isSpawnUrl">true</entry>
+    <!-- 有序爬取中，结束页页码-->
+    <entry key="endInPageIndex">10</entry>
+    <!-- 页面下载超时，重试次数-->
+    <entry key="retryTimes">5</entry>
+    <!-- 下载时，连接超时等待时间-->
+    <entry key="timeOut">10000</entry>
+    <!-- 单位是毫秒-->
+    <entry key="sleepTimeThread">0</entry>
+    <!-- 循环重试次数-->
+    <entry key="cycleRetryTimes">5</entry>
+    <!-- 请求头的UserAgent信息-->
+    <entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
+    <!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
+    <entry key="sleepTimeSpider">4000</entry>
+    <!-- 起始种子URL -->
+    <entry key="startUrl">http://bbs.csdn.net/tech_hot_topics?page=1</entry>
+    <!-- 列表页的规则 -->
+    <entry key="urlList"></entry>
+    <!-- 详情页链接抽取规则 -->
+    <entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
+    <!--  -->
+    <entry key="offset">0</entry>
+    <!--  -->
+    <entry key="limitLine">10</entry>
+    <!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
+    <entry key="noUrlWaitTime">10000</entry>
+    <!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
+    <entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
+    <!-- 列表页url规则的前缀 -->
+    <entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
+    <!-- 列表页url规则的后缀 -->
+    <entry key="postfixUrl"></entry>
+    <!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有	XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
+    <entry key="extractMethod">CSSPath</entry>
+    <!-- 代理ip -->
+    <entry key="proxyIp">192.168.245.1</entry>
+    <!-- 代理端口号 -->
+    <entry key="proxyPort">3128</entry>
+    <!-- 爬取模式 	0:mirror	或者是 	1:increment-->
+    <entry key="mode">0</entry>
+    <!-- 增量模式观察的页数 -->
+    <entry key="incrementalPages">50</entry>
+    <!-- 列表页更新完后休息时间	单位 为秒 为了给足够时间下载更新的详情页 -->
+    <entry key="incrementSleepTime">20000</entry>
+    
+    <!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
+    <entry key="minInterval">10000</entry>
+    <entry key="maxInterval">20000</entry>
+    
+    <!--长时间睡眠时候的睡眠时间的上下限 -->
+    <entry key="minlongSleepTime">1800000</entry>
+    <entry key="MaxlongSleepTime">7200000</entry>
+    
+    <!--长时间睡眠时候的睡眠间隔的上下限 -->
+    <entry key="minlongSleepInterval">3600000</entry>
+    <entry key="MaxlongSleepInterval">7200000</entry>
+    
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sites/slashdot.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>66</endInPageIndex>
+
+	    		<urlPost> #firehoselist>article>header>h2>span>a</urlPost>
+	    		<fixAllRelativeHrefs>http://slashdot.org</fixAllRelativeHrefs>
+ 	    		<prefixUrl>http://slashdot.org/?page=</prefixUrl>
+	   		<postfixUrl></postfixUrl>
+	   		<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>slashdot</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+
+</site>
+

crawler/dailyScheduledCrawler/fetch_networks/sites/softpedia.xml

@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>linux</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://linux.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://linux.softpedia.com/index</prefixUrl>
+	   			<postfixUrl>.shtml</postfixUrl>
+	   			<incrementalPages>3</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>win</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://win.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://win.softpedia.com/index</prefixUrl>
+	   			<postfixUrl>.shtml</postfixUrl>
+	   			<incrementalPages>3</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>mac</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>35</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://mac.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://mac.softpedia.com/index</prefixUrl>
+	   			<postfixUrl>.shtml</postfixUrl>
+	   			<incrementalPages>5</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>windows-phone</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>46</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://mobile.softpedia.com/windows-phone,</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>5</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>android</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>35</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://mobile.softpedia.com/android,</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>ios</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>22</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://mobile.softpedia.com/ios,</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>1</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>driver</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://drivers.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://drivers.softpedia.com/index</prefixUrl>
+	    		<postfixUrl>.shtml</postfixUrl>
+	    		<incrementalPages>10</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	 </subSite>
+	    	<subSite>
+	    	<!-- 普通网站name不用写 -->
+	    		<name>game</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://games.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://games.softpedia.com/index</prefixUrl>
+	    		<postfixUrl>.shtml</postfixUrl>
+	    		<incrementalPages>2</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+		</subSite>
+		<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name>webscripts</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>30</endInPageIndex>
+	    		<urlPost>h4>a</urlPost>
+	    		<fixAllRelativeHrefs>http://webscripts.softpedia.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://webscripts.softpedia.com/index/latest-scripts-3</prefixUrl>
+	    		<postfixUrl>-0-0.html</postfixUrl>
+	    		<incrementalPages>3</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	   	</subSite>
+	 
+	</subSites>
+    
+	<properties>
+	    <domain>softpedia</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/sourceforge.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>17621</endInPageIndex>
+	    		<urlPost>ul.projects>li>div.project_info>header>a</urlPost>
+	    		<fixAllRelativeHrefs>http://sourceforge.net</fixAllRelativeHrefs>
+
+	    		<!-- <prefixUrl>http://sourceforge.net/directory/?page=</prefixUrl>-->
+
+	    		 <prefixUrl>http://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update&amp;page=</prefixUrl>
+	    		
+	   			<postfixUrl></postfixUrl>
+	   			 <!--<incrementalPages>300</incrementalPages>
+	    		<incrementSleepTime>250000</incrementSleepTime>--> 
+	    		<incrementalPages>8</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>sourceforge</domain>
+	    <minInterval>10000</minInterval>
+	    <maxInterval>20000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/sites/spider_test.xml

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--  <!DOCTYPE  site[
+  <!ELEMENT site (subSites,properties)>
+  <!ELEMENT subSites (subSite+)>
+  <!ELEMENT properties (entry+)>
+  <!ELEMENT subSite (name,startPageIndex,endInPageIndex,urlPost,fixAllRelativeHrefs,prefixUrl,postfixUrl,incrementalPages,incrementSleepTime,mode)>
+  <!ELEMENT comment   (#PCDATA)>
+  <!ELEMENT name   (#PCDATA)>
+  <!ELEMENT starPageIndex   (#PCDATA)>
+  <!ELEMENT endInPageIndex   (#PCDATA)>
+  <!ELEMENT urlPost   (#PCDATA)>
+  <!ELEMENT fixAllRelativeHrefs   (#PCDATA)>
+  <!ELEMENT prefixUrl   (#PCDATA)>
+  <!ELEMENT postfixUrl   (#PCDATA)>
+  <!ELEMENT incrementalPages   (#PCDATA)>
+  <!ELEMENT incrementSleepTime   (#PCDATA)> 
+  <!ELEMENT mode   (#PCDATA)> 
+  <!ELEMENT entry   (#PCDATA)> 
+  <!ATTLIST entry key CDATA "ID">
+]>  -->
+
+<!--  <!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">-->
+
+<site>
+
+
+	<subSites>
+	    	<subSite>
+	    		<name>database</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>20</endInPageIndex>
+	    		<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
+	    		<fixAllRelativeHrefs>http://localhost:8080/SpiderTest</fixAllRelativeHrefs>
+	    		<prefixUrl>http://localhost:8080/SpiderTest/database?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>15</incrementalPages>
+	    		<incrementSleepTime>300</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	    	<subSite>
+	    		<name>os</name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>20</endInPageIndex>
+	    		<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
+	    		<fixAllRelativeHrefs>http://localhost:8080/SpiderTest</fixAllRelativeHrefs>
+	    		<prefixUrl>http://localhost:8080/SpiderTest/os?page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>15</incrementalPages>
+	    		<incrementSleepTime>300</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>SpiderTes</domain>
+	    <minInterval>1000</minInterval>
+	    <maxInterval>3000</maxInterval>
+	    <minlongSleepTime>1800000</minlongSleepTime>
+	    <maxlongSleepTime>7200000</maxlongSleepTime>
+	    <minlongSleepInterval>3600000</minlongSleepInterval>
+	    <maxlongSleepInterval>7200000</maxlongSleepInterval>
+	</properties>
+
+</site>
+
+ 

crawler/dailyScheduledCrawler/fetch_networks/sites/stackoverflow.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site>
+	<subSites>
+	    	<subSite>
+	    		<!-- 普通网站name不用写 -->
+	    		<name></name>
+	    		<startPageIndex>1</startPageIndex>
+	    		<endInPageIndex>25000</endInPageIndex>
+	    		<urlPost>div#mainbar>div#questions>div.question-summary>div.summary>h3>a</urlPost>
+	    		<fixAllRelativeHrefs>http://stackoverflow.com</fixAllRelativeHrefs>
+	    		<prefixUrl>http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=</prefixUrl>
+	   			<postfixUrl></postfixUrl>
+	   			<incrementalPages>250</incrementalPages>
+	    		<incrementSleepTime>86400</incrementSleepTime>
+	    		<mode>0</mode>
+	    	</subSite>
+	 </subSites>
+    
+	<properties>
+	    <domain>stackoverflow</domain>
+	    <minInterval>2000</minInterval>
+	    <maxInterval>3000</maxInterval>
+	    <minlongSleepTime>900000</minlongSleepTime>
+	    <maxlongSleepTime>1800000</maxlongSleepTime>
+	    <minlongSleepInterval>7200000</minlongSleepInterval>
+	    <maxlongSleepInterval>14400000</maxlongSleepInterval>
+	</properties>
+
+</site>

crawler/dailyScheduledCrawler/fetch_networks/spider.conf.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
+<properties>
+<comment>Spider</comment>
+<entry key="pageF">1</entry>
+<entry key="domain">ques_oschina</entry>
+<entry key="threadNum">2</entry>
+<entry key="startPageIndex">1</entry>
+<entry key="isSpawnUrl">true</entry>
+<entry key="endInPageIndex">2</entry>
+<entry key="retryTimes">5</entry>
+<entry key="timeOut">10000</entry>
+<entry key="sleepTimeThread">5000</entry>
+<entry key="cycleRetryTimes">5</entry>
+<entry key="userAgent">Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36</entry>
+<entry key="sleepTimeSpider">10000</entry>
+</properties>

crawler/dailyScheduledCrawler/fetch_networks/sql/detail_html.sql

@@ -0,0 +1,15 @@
+(
+`id`  int(11) NOT NULL AUTO_INCREMENT ,
+`url`  varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
+`html`  mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
+`crawledTime`  datetime NULL DEFAULT NULL ,
+`urlMd5`  varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
+`pageMd5`  varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
+`history`  varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
+PRIMARY KEY (`id`)
+)
+ENGINE=InnoDB
+DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
+AUTO_INCREMENT=1
+ROW_FORMAT=COMPACT
+;