linchun
/
ossean
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: linchun:master
Branches Tags
linchun:master
linchun:wt
linchun:pre
linchun:develop
..
compare: linchun:pre
Branches Tags
linchun:master
linchun:wt
linchun:pre
linchun:develop

No commits in common. "master" and "pre" have entirely different histories.
master ... pre

2119 changed files with 189077 additions and 26242 deletions
Show Stats Expand all files Collapse all files

11
.gitignore vendored
View File

@ -72,6 +72,7 @@
/trustie2/files/*
/trustie2/log/*
/trustie2/tmp/*
/trustie2/public/images/avatars/*
/trustie2/Gemfile.lock
/trustie2/db/schema.rb
!/trustie2/solr/conf
@ -108,12 +109,4 @@
/gather_program/target/
/gather_program/log/*
/gather_program/.settings/
/gather_program/.classpath
/crawler/dailyScheduledCrawler/fetch_networks/target/
/crawler/dailyScheduledCrawler/fetch_networks/log/*
/crawler/daily_scheduler/log/*
/crawler/daily_scheduler/log_mem/*
/crawler/moreSmarterCrawler/fetch_networks/target/
/crawler/moreSmarterCrawler/fetch_networks/log/*
/gather_program/.classpath

29
crawler/dailyScheduledCrawler/fetch_networks/.project
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>fetchnetworks</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.maven.ide.eclipse.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.maven.ide.eclipse.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

5
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs
View File

@ -1,5 +0,0 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8

2
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs
View File

@ -1,2 +0,0 @@
eclipse.preferences.version=1
line.separator=\n

14
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs
View File

@ -1,14 +0,0 @@
#Sun Jan 04 15:44:05 CST 2015
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.7

4
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs
View File

@ -1,4 +0,0 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

30
crawler/dailyScheduledCrawler/fetch_networks/bin/51cto_blog.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='51cto_blog'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/apache.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='apache'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/bytes.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='bytes'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_news.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='cnblogs_news'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_solved.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='cnblogs_q_solved'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='cnblogs_q_unsolved'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/code_project.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='code_project'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_ask.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='csdn_ask'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_blog.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='csdn_blog'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_topic.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='csdn_topic'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx256m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/dewen_question.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='dewen_question'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/freecode.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='freecode'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/gna.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='gna'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_post.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='ibm_post'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_project.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='ibm_project'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_ask.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='iteye_ask'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_blog.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='iteye_blog'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/lagou.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='lagou'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/linuxtone.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='linuxtone'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/lupaworld.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='lupaworld'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/neitui.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='neitui'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/openhub.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='openhub'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_project.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='oschina_project'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_question.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='oschina_question'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/phpchina.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='phpchina'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

79
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/log4j.xml
View File

@ -1,79 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="threshold" value="ERROR" />
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="./log/error.log" />
<param name="threshold" value="ERROR" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<appender name="file_log" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="./log/webmagic.log" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<!-- 邮件（只有ERROR时才会发送！） -->
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
<param name="threshold" value="debug" />
<!-- 日志的错误级别 <param name="threshold" value="fatal"/> -->
<!-- 缓存文件大小，日志达到512K时发送Email -->
<param name="BufferSize" value="1" />
<!-- 单位K -->
<param name="From" value="ossean_debug@163.com" />
<param name="SMTPHost" value="smtp.163.com" />
<param name="Subject" value="ossean-crawler-debug-log4jMessage" />
<param name="To" value="gcm3651@126.com" />
<param name="SMTPUsername" value="ossean_debug" />
<param name="SMTPPassword" value="goodwell123" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss.SSS} [%p]-[%c] %m%n" />
</layout>
</appender>
<!-- 数据库状态 -->
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
<param name="URL"
value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8" />
<param name="driver" value="com.mysql.jdbc.Driver" />
<param name="user" value="root" />
<param name="password" value="1234" />
<param name="sql"
value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%p','%c','%m','%l')" />
<!-- <layout class="org.apache.log4j.PatternLayout"> <param name="ConversionPattern"
value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES
('%d{yyyy-MM-dd HH:mm:ss}','%t','%.50p','%.50c','%.1000m','%.50l')" /> </layout> -->
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都 FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="LevelMin" value="DEBUG" />
<param name="LevelMax" value="FATAL" />
</filter>
</appender>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" />
<appender-ref ref="stdout" />
<!-- <appender-ref ref="file" /> -->
<!-- <appender-ref ref="file_log" /> -->
<!-- <appender-ref ref="MAIL" /> -->
<!-- <appender-ref ref="DATABASE" /> -->
</root>
</log4j:configuration>

32
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource" />
</bean>
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="net.trustie.webmagic" />
</bean>
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://192.168.80.104:3306/pages?characterEncoding=UTF-8" />
<property name="username" value="influx" />
<property name="password" value="influx1234" />
<property name="timeBetweenEvictionRunsMillis">
<value>21600000</value><!--6 hours-->
</property>
<property name="minEvictableIdleTimeMillis">
<value>21600000</value><!--connection的空闲时间大于这个值,就直接被关闭,并从连接池中删除-->
</property>
</bean>
</beans>

16
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext.xml
View File

@ -1,16 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
<context:annotation-config/>
<context:component-scan base-package="net.trustie.webmagic"/>
</beans>

43
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/testDBlogger.xml
View File

@ -1,43 +0,0 @@
<!-- 输出通道"DATABASE"，输出方式是：输出所有级别的LOG到数据库 -->
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
<!--数据库的驱动，这里用的是MSSQL-->
<param name="driver" value="com.microsoft.jdbc.sqlserver.SQLServerDriver"/>
<!--这是Oracle用的驱动，在这里用MSSQL所以屏蔽掉了
<param name="driver" value="oracle.jdbc.driver.OracleDriver" />
-->
<!--要连接的数据库-->
<param name="URL" value="jdbc:microsoft:sqlserver://192.168.0.120:
1433;DatabaseName=test" />
<!--连接数据库的用户名-->
<param name="user" value="sa" />
<!--连接数据库的密码-->
<param name="password" value="sa" />
<!--向MSSQL数据库表LOG中插入数据的sql语句-->
<param name="sql" value=" INSERT INTO LOG
(LOGDATE,LOGLEVEL,LOGCLASS,LOGLOGGER,LOGMESSAGE)
values ('%d{yyyy-MM-dd HH:mm:ss}','%.50p', '%.50c',
'%.50l', '%.1000m')"/>
<!-- Oracle 的 insert 语句
<param name="sql" value=" insert into logrecord
(id,packageid,userid,syscodeid,info,logtime,loglevel)
values(?,?,?,?,?,to_date('%d{yyyy-MM-dd HH:mm:ss}',
'yyyy-MM-dd HH24:mi:ss'),'%p')" />
-->
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都
FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="LevelMin" value="ERROR"/>
<param name="LevelMax" value="FATAL"/>
</filter>
</appender>

30
crawler/dailyScheduledCrawler/fetch_networks/bin/sample_start.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='freecode'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/slashdot.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='slashdot'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/softpedia.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='softpedia'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/sourceforge.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='sourceforge'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/stackoverflow.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='stackoverflow'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn48m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

15
crawler/dailyScheduledCrawler/fetch_networks/bin/start_all.sh
View File

@ -1,15 +0,0 @@
#!/bin/bash
sh bin/cnblogs_news.sh
sh bin/cnblogs_q_solved.sh
sh bin/cnblogs_q_unsolved.sh
sh bin/csdn_ask.sh
sh bin/csdn_blog.sh
sh bin/csdn_topic.sh
sh bin/dewen_question.sh
sh bin/iteye_ask.sh
sh bin/openhub.sh
sh bin/oschina_project.sh
sh bin/oschina_question.sh
sh bin/sourceforge.sh
sh bin/stackoverflow.sh

BIN
crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-core-fix.jar
View File

Binary file not shown.

25
crawler/dailyScheduledCrawler/fetch_networks/ossean-bak.xml
View File

@ -1,25 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<entry key="pageF">1</entry>
<entry key="domain">one</entry>
<entry key="threadNum">5</entry>
<entry key="startPageIndex">1</entry>
<entry key="isSpawnUrl">true</entry>
<entry key="endInPageIndex">20000</entry>
<entry key="retryTimes">5</entry>
<entry key="timeOut">30000</entry>
<entry key="sleepTimeThread">5000</entry>
<entry key="cycleRetryTimes">5</entry>
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<entry key="sleepTimeSpider">10000</entry>
<entry key="startUrl">http://www.oschina.net/question/</entry>
<entry key="urlList">http://www\.oschina\.net/question\?catalog=[12]\&show=active\&p=\d+</entry>
<entry key="urlPost">http://www\.oschina\.net/question/\d+_\d+</entry>
<entry key="offset">0</entry>
<entry key="limitLine">10</entry>
<entry key="noUrlWaitTime">10000</entry>
<entry key="postTableName">html_test</entry>
<entry key="listTableName">target_url</entry>
</properties>

202
crawler/dailyScheduledCrawler/fetch_networks/pom.xml
View File

@ -1,202 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>net.trustie</groupId>
<artifactId>fetchnetworks</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- m2eclipse wtp 0.12+ enabled to configure contextRoot, add by w.vela -->
<m2eclipse.wtp.contextRoot>/</m2eclipse.wtp.contextRoot>
<spring-version>3.1.1.RELEASE</spring-version>
<spring-security-version>3.1.0.RELEASE</spring-security-version>
</properties>
<build>
<sourceDirectory>${basedir}/src/main/java</sourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>xerces</groupId>
<artifactId>xerces</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20141113</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>xsoup</artifactId>
<version>0.2.4</version>
</dependency>
<dependency>
<groupId>com.github.dreamhead</groupId>
<artifactId>moco-core</artifactId>
<version>0.10.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.7</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>1.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>0.8.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.4</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.18</version>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${spring-version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.trustie</groupId>
<artifactId>webmagic-core-fix</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/webmagic-core-fix.jar</systemPath>
</dependency>
<dependency>
<groupId>net.trustie</groupId>
<artifactId>webmagic-extension-fix</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/webmagic-extension-fix.jar</systemPath>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>2.42.2</version>
</dependency>
</dependencies>
</project>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/51cto_blog.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>51cto</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>div.rightbox>div.r_li>h4>a</urlPost>
<fixAllRelativeHrefs>http://blog.51cto.com</fixAllRelativeHrefs>
<prefixUrl>http://blog.51cto.com/original/0/</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>51cto_blog</domain>
<minInterval>20000</minInterval>
<maxInterval>30000</maxInterval>
<minlongSleepTime>600000</minlongSleepTime>
<maxlongSleepTime>1200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

34
crawler/dailyScheduledCrawler/fetch_networks/sites/Copy of sample_site.xml
View File

@ -1,34 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">csdn_topic</entry>
<!-- 有序爬取中，起始页页码-->
<entry key="startPageIndex">1</entry>
<!-- 有序爬取中，结束页页码-->
<entry key="endInPageIndex">10</entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
<entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">50</entry>
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
<!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

30
crawler/dailyScheduledCrawler/fetch_networks/sites/apache.xml
View File

@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1</endInPageIndex>
<urlPost>table>tbody>tr>td.body>div>div>ul>li> a</urlPost>
<fixAllRelativeHrefs>http://projects.apache.org</fixAllRelativeHrefs>
<prefixUrl>http://projects.apache.org/indexes/alpha.html</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>864000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>apache</domain>
<minInterval>10000</minInterval>
<maxInterval>30000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">51cto_blog</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">30</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://blog.51cto.com/original/0/1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.rightbox>div.r_li>h4>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://blog.51cto.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://blog.51cto.com/original/0/</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">16</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of apache.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">apache</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">30</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://projects.apache.org/indexes/alpha.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table>tbody>tr>td.body>div>div>ul>li> a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://projects.apache.org/indexes/alpha.html</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://projects.apache.org/indexes/alpha.html</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">1</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">8640000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">cnblogs_news</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">4845</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://news.cnblogs.com/n/page/1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#news_list>div.news_block>div.content>h2.news_entry>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://news.cnblogs.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://news.cnblogs.com/n/page/</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">5</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">cnblogs_q_solved</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">160</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">4000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://q.cnblogs.com/list/solved?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://q.cnblogs.com/list/solved?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

72
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml
View File

@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">cnblogs_q_unsolved</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">160</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">4000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://q.cnblogs.com/list/unsolved?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://q.cnblogs.com/list/unsolved?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of code_project.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">codeproject</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">21</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content-list-item>div.title>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.codeproject.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">1</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">dewen_question</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">1114</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.dewen.io/questions?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.dewen.io</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.dewen.io/questions?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">1</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">432000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of gna.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">gna</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">1348</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=1&amp;max_rows=1#results</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table>tbody>tr>td>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://gna.org</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">&amp;max_rows=1#results</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">200</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">432000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">lupaworld</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">2</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">479</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.lupaworld.com/forum-13746-2.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table#threadlisttableid>tbody>tr>th>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.lupaworld.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.lupaworld.com/forum-13746-</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.html</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">2592000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of openhub.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- -->
<entry key="domain">openhub</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">66784</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">60000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">https://www.openhub.net/p?page=1&amp;q=&amp;sort=users</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.project>h2.title>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">60000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">https://www.openhub.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">https://www.openhub.net/p?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">&amp;q=&amp;sort=users</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">500</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">100000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of phpchina.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">phpchina</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">1289</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://bbs.phpchina.com/forum-17-1.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table#threadlisttableid>tbody>tr>th>a.s.xst</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://bbs.phpchina.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.phpchina.com/forum-17-</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.html</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">1296000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of softpedia.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">softpedia</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">30</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://linux.softpedia.com/index1.shtml</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#sjmp>div>h4>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://linux.softpedia.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://linux.softpedia.com/index</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.shtml</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">10</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- -->
<entry key="domain">sourceforge</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">17621</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">60000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">10000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://sourceforge.net/directory/?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">ul.projects>li>div.project_info>header>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://sourceforge.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://sourceforge.net/directory/?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">500</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">250000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- 有序爬取时，每次生成页面链接数目-->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">stackoverflow</entry>
<!-- 线程数目-->
<entry key="threadNum">1</entry>
<!-- 有序爬取中，起始页页码-->
<entry key="startPageIndex">1</entry>
<!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
<entry key="isSpawnUrl">true</entry>
<!-- 有序爬取中，结束页页码-->
<entry key="endInPageIndex">25000</entry>
<!-- 页面下载超时，重试次数-->
<entry key="retryTimes">5</entry>
<!-- 下载时，连接超时等待时间-->
<entry key="timeOut">60000</entry>
<!-- 超时时间，单位是毫秒-->
<entry key="sleepTimeThread">0</entry>
<!-- 循环重试次数-->
<entry key="cycleRetryTimes">5</entry>
<!-- 请求头的UserAgent信息-->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
<entry key="sleepTimeSpider">10000</entry>
<!-- 起始种子URL -->
<entry key="startUrl">http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#mainbar>div#questions>div.question-summary>div.summary>h3>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
<entry key="noUrlWaitTime">10000</entry>
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
<entry key="fixAllRelativeHrefs">http://stackoverflow.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">250</entry>
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
<entry key="minInterval">2000</entry>
<entry key="maxInterval">3000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">2400000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">7200000</entry>
<entry key="MaxlongSleepInterval">14400000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">linuxtone</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">64</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://bbs.linuxtone.org/forum-15-1.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table>tbody>tr>th>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://bbs.linuxtone.org</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.linuxtone.org/forum-15-</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.html</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">2592000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">iteye_blog</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">15</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.iteye.com/blogs?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content>h3>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.iteye.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.iteye.com/blogs?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/bytes.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>question</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2000</endInPageIndex>
<urlPost>table#threadslist>tbody>tr>td>div>div.threadbit_title>a</urlPost>
<fixAllRelativeHrefs>http://bytes.com</fixAllRelativeHrefs>
<prefixUrl>http://bytes.com/answers/</prefixUrl>
<postfixUrl>/</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>bytes</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

32
crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_news.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>4845</endInPageIndex>
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://news.cnblogs.com</fixAllRelativeHrefs>
<prefixUrl>http://news.cnblogs.com/n/page/</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>5</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>cnblogs_news</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

27
crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_solved.xml
View File

@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>160</endInPageIndex>
<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
<prefixUrl>http://q.cnblogs.com/list/solved?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>cnblogs_q_solved</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
<prefixUrl>http://q.cnblogs.com/list/unsolved?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>cnblogs_q_unsolved</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/code_project.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>259</endInPageIndex>
<urlPost>div.content-list-item>div.title>a</urlPost>
<fixAllRelativeHrefs>http://www.codeproject.com</fixAllRelativeHrefs>
<!--<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</prefixUrl>-->
<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=366&amp;la_app=20&amp;la_minscore=1.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>codeproject</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_ask.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2496</endInPageIndex>
<urlPost>div.common_con>div.questions_detail_con>dl>dt>a</urlPost>
<fixAllRelativeHrefs>http://ask.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://ask.csdn.net/?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>25</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>csdn_ask</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_blog.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>101</endInPageIndex>
<urlPost>div.main_center>div.blog_list>h1>a:not(.category)</urlPost>
<fixAllRelativeHrefs>http://blog.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://blog.csdn.net/index.html?&amp;page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>csdn_blog</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

81
crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_topic.xml
View File

@ -1,81 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>mobile</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>808</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/Mobile?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>8</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>CloudComputing</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>22</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/CloudComputing?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>java</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2555</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/Java?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>4</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>web</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1853</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/WebDevelop?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>OtherLanguage</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>336</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/OtherLanguage?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>csdn_topic</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

30
crawler/dailyScheduledCrawler/fetch_networks/sites/dewen_question.xml
View File

@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1114</endInPageIndex>
<urlPost>div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</urlPost>
<fixAllRelativeHrefs>http://www.dewen.io</fixAllRelativeHrefs>
<prefixUrl>http://www.dewen.io/questions?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>432000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>dewen_question</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

59
crawler/dailyScheduledCrawler/fetch_networks/sites/freecode.xml
View File

@ -1,59 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">5</entry>
<!-- -->
<entry key="domain">freecode</entry>
<!-- -->
<entry key="threadNum">5</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">4799</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">30000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">10000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.freecode.com/projects?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.project>h2.release-head>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">60000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.freecode.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.freecode.com/projects?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">5</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
</properties>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/gna.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1348</endInPageIndex>
<urlPost>table>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://gna.org</fixAllRelativeHrefs>
<prefixUrl>http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=</prefixUrl>
<postfixUrl>&amp;max_rows=1#results</postfixUrl>
<incrementalPages>100</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>gna</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_post.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>blog</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>138</endInPageIndex>
<urlPost>table>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
<prefixUrl>http://www.ibm.com/developerworks/cn/views/opensource/libraryview.jsp?site_id=10&amp;contentarea_by=%E6%89%80%E6%9C%89%E4%B8%93%E5%8C%BA&amp;sort_by=&amp;sort_order=2&amp;start=</prefixUrl>
<postfixUrl>00&amp;end=13874&amp;topic_by=-1&amp;product_by=&amp;type_by=%E6%89%80%E6%9C%89%E7%B1%BB%E5%88%AB&amp;show_abstract=false&amp;search_by=&amp;industry_by=&amp;series_title_by=</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>6048000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>ibm_post</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_project.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1</endInPageIndex>
<urlPost>table>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
<prefixUrl>http://www.ibm.com/developerworks/views/opensource/projects.jsp</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>864000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>ibm_project</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

59
crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_ask.xml
View File

@ -1,59 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">5</entry>
<!-- -->
<entry key="domain">iteye_ask</entry>
<!-- -->
<entry key="threadNum">5</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">5329</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">30000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.iteye.com/ask?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#ask_list>div.question-summary>div.summary>h3>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">60000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.iteye.com/</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.iteye.com/ask?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">50</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
</properties>

78
crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_bbs.xml
View File

@ -1,78 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name>mobile</name>
<startPageIndex>z</startPageIndex>
<endInPageIndex>111</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/mobile?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>language</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>242</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/language?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>web</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>320</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/web?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>java</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>754</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/Java?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>tech</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>242</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/Tech?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>iteye_bbs</domain>
<minInterval>30000</minInterval>
<maxInterval>50000</maxInterval>
<minlongSleepTime>3600000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

28
crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_blog.xml
View File

@ -1,28 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name>blog</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>15</endInPageIndex>
<urlPost>div.content>h3>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/blogs?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>iteye_blog</domain>
<minInterval>30000</minInterval>
<maxInterval>50000</maxInterval>
<minlongSleepTime>3600000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/javaforge.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">javaforge</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">31</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://javaforge.com/listProjects.spr?page=1&amp;displayAllProjects=on</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table#entry>tbody>tr>td:nth-child(1) > a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://javaforge.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://javaforge.com/listProjects.spr?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">&amp;displayAllProjects=on</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">10</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/lagou.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>houduan</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost> #container > div.content>ul> li> div> div > a</urlPost>
<fixAllRelativeHrefs>http://www.lagou.com</fixAllRelativeHrefs>
<prefixUrl>http://www.lagou.com/jobs/list_%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91?kd=%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91&amp;spc=1&amp;pl=&amp;gj=&amp;xl=&amp;yx=&amp;gx=&amp;st=&amp;labelWords=label&amp;lc=&amp;workAddress=&amp;city=%E5%85%A8%E5%9B%BD&amp;requestId=&amp;pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>lagou</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

81
crawler/dailyScheduledCrawler/fetch_networks/sites/linuxtone.xml
View File

@ -1,81 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>System</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>64</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-15-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>server</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-22-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>dbserver</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>32</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-24-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>yngz</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>32</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-49-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>fzjh</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>24</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-26-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>linuxtone</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

31
crawler/dailyScheduledCrawler/fetch_networks/sites/lupaworld.xml
View File

@ -1,31 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>479</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://www.lupaworld.com</fixAllRelativeHrefs>
<prefixUrl>http://www.lupaworld.com/forum-13746-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>lupaworld</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/neitui.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>610</endInPageIndex>
<urlPost> ul>li>div.cont>div.jobnote.clearfix > div.jobnote-l>a</urlPost>
<fixAllRelativeHrefs>http://www.neitui.me</fixAllRelativeHrefs>
<prefixUrl>http://www.neitui.me/neitui/type=all&amp;page=</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>neitui</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

152
crawler/dailyScheduledCrawler/fetch_networks/sites/open_open.xml
View File

@ -1,152 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>dev</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>370</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/1?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>language</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>652</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/36?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>website</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>41</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/57?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>server</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>145</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/75?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>database</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>102</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/90?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>plugin</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>53</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/52?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>OS</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>42</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/155?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>manage</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>12</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/319?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>dev_manage</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>9</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/159?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>util</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>7</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/322?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>open_open</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

35
crawler/dailyScheduledCrawler/fetch_networks/sites/openhub.xml
View File

@ -1,35 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>66784</endInPageIndex>
<urlPost>div.well>h2.title>a</urlPost>
<fixAllRelativeHrefs>https://www.openhub.net</fixAllRelativeHrefs>
<!--<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
<postfixUrl>&amp;q=&amp;sort=users</postfixUrl>
<incrementalPages>300</incrementalPages>
<incrementSleepTime>300000</incrementSleepTime> -->
<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
<postfixUrl>&amp;q=&amp;sort=new</postfixUrl>
<incrementalPages>8</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>openhub</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_project.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2044</endInPageIndex>
<urlPost>div.ProjectList>ul.List>li>h3>a</urlPost>
<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
<prefixUrl>http://www.oschina.net/project/list?prefix=&amp;sort=time&amp;p=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>oschina_project</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

31
crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_question.xml
View File

@ -1,31 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2361</endInPageIndex>
<urlPost>ul.list>li.question>div.question-detail>strong>a</urlPost>
<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
<prefixUrl>http://www.oschina.net/question?catalog=1&amp;show=active&amp;p=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>oschina_question</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

59
crawler/dailyScheduledCrawler/fetch_networks/sites/ossean.xml
View File

@ -1,59 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">3</entry>
<!-- -->
<entry key="domain">ossean</entry>
<!-- -->
<entry key="threadNum">3</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">5777</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">500</entry>
<!-- 起始页 -->
<entry key="startUrl">http://localhost/open_source_projects?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">ul.projects>li.project-table>div.root>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://localhost</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://localhost/open_source_projects?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">5</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">0</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/phpchina.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>ask</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1289</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-17-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>server_os</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>91</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-195-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>web</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>81</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-213-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>db</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>43</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-196-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>phpchina</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

72
crawler/dailyScheduledCrawler/fetch_networks/sites/sample_site.xml
View File

@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- 有序爬取时，每次生成页面链接数目-->
<entry key="pageF">3</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">csdn_topic</entry>
<!-- 线程数目-->
<entry key="threadNum">3</entry>
<!-- 有序爬取中，起始页页码-->
<entry key="startPageIndex">1</entry>
<!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
<entry key="isSpawnUrl">true</entry>
<!-- 有序爬取中，结束页页码-->
<entry key="endInPageIndex">10</entry>
<!-- 页面下载超时，重试次数-->
<entry key="retryTimes">5</entry>
<!-- 下载时，连接超时等待时间-->
<entry key="timeOut">10000</entry>
<!-- 单位是毫秒-->
<entry key="sleepTimeThread">0</entry>
<!-- 循环重试次数-->
<entry key="cycleRetryTimes">5</entry>
<!-- 请求头的UserAgent信息-->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
<entry key="sleepTimeSpider">4000</entry>
<!-- 起始种子URL -->
<entry key="startUrl">http://bbs.csdn.net/tech_hot_topics?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
<entry key="noUrlWaitTime">10000</entry>
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
<entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">50</entry>
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
<!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

32
crawler/dailyScheduledCrawler/fetch_networks/sites/slashdot.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>66</endInPageIndex>
<urlPost> #firehoselist>article>header>h2>span>a</urlPost>
<fixAllRelativeHrefs>http://slashdot.org</fixAllRelativeHrefs>
<prefixUrl>http://slashdot.org/?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>slashdot</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

134
crawler/dailyScheduledCrawler/fetch_networks/sites/softpedia.xml
View File

@ -1,134 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>linux</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://linux.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://linux.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>win</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://win.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://win.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>mac</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>35</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mac.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mac.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>5</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>windows-phone</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>46</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mobile.softpedia.com/windows-phone,</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>5</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>android</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>35</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mobile.softpedia.com/android,</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>ios</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>22</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mobile.softpedia.com/ios,</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>driver</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://drivers.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://drivers.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>10</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>game</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://games.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://games.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>webscripts</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://webscripts.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://webscripts.softpedia.com/index/latest-scripts-3</prefixUrl>
<postfixUrl>-0-0.html</postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>softpedia</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

35
crawler/dailyScheduledCrawler/fetch_networks/sites/sourceforge.xml
View File

@ -1,35 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>17621</endInPageIndex>
<urlPost>ul.projects>li>div.project_info>header>a</urlPost>
<fixAllRelativeHrefs>http://sourceforge.net</fixAllRelativeHrefs>
<!-- <prefixUrl>http://sourceforge.net/directory/?page=</prefixUrl>-->
<prefixUrl>http://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update&amp;page=</prefixUrl>
<postfixUrl></postfixUrl>
<!--<incrementalPages>300</incrementalPages>
<incrementSleepTime>250000</incrementSleepTime>-->
<incrementalPages>8</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>sourceforge</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

66
crawler/dailyScheduledCrawler/fetch_networks/sites/spider_test.xml
View File

@ -1,66 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- <!DOCTYPE site[
<!ELEMENT site (subSites,properties)>
<!ELEMENT subSites (subSite+)>
<!ELEMENT properties (entry+)>
<!ELEMENT subSite (name,startPageIndex,endInPageIndex,urlPost,fixAllRelativeHrefs,prefixUrl,postfixUrl,incrementalPages,incrementSleepTime,mode)>
<!ELEMENT comment (#PCDATA)>
<!ELEMENT name (#PCDATA)>
<!ELEMENT starPageIndex (#PCDATA)>
<!ELEMENT endInPageIndex (#PCDATA)>
<!ELEMENT urlPost (#PCDATA)>
<!ELEMENT fixAllRelativeHrefs (#PCDATA)>
<!ELEMENT prefixUrl (#PCDATA)>
<!ELEMENT postfixUrl (#PCDATA)>
<!ELEMENT incrementalPages (#PCDATA)>
<!ELEMENT incrementSleepTime (#PCDATA)>
<!ELEMENT mode (#PCDATA)>
<!ELEMENT entry (#PCDATA)>
<!ATTLIST entry key CDATA "ID">
]> -->
<!-- <!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">-->
<site>
<subSites>
<subSite>
<name>database</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>20</endInPageIndex>
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://localhost:8080/SpiderTest</fixAllRelativeHrefs>
<prefixUrl>http://localhost:8080/SpiderTest/database?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>15</incrementalPages>
<incrementSleepTime>300</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>os</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>20</endInPageIndex>
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://localhost:8080/SpiderTest</fixAllRelativeHrefs>
<prefixUrl>http://localhost:8080/SpiderTest/os?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>15</incrementalPages>
<incrementSleepTime>300</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>SpiderTes</domain>
<minInterval>1000</minInterval>
<maxInterval>3000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/stackoverflow.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>25000</endInPageIndex>
<urlPost>div#mainbar>div#questions>div.question-summary>div.summary>h3>a</urlPost>
<fixAllRelativeHrefs>http://stackoverflow.com</fixAllRelativeHrefs>
<prefixUrl>http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>250</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>stackoverflow</domain>
<minInterval>2000</minInterval>
<maxInterval>3000</maxInterval>
<minlongSleepTime>900000</minlongSleepTime>
<maxlongSleepTime>1800000</maxlongSleepTime>
<minlongSleepInterval>7200000</minlongSleepInterval>
<maxlongSleepInterval>14400000</maxlongSleepInterval>
</properties>
</site>

17
crawler/dailyScheduledCrawler/fetch_networks/spider.conf.xml
View File

@ -1,17 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<entry key="pageF">1</entry>
<entry key="domain">ques_oschina</entry>
<entry key="threadNum">2</entry>
<entry key="startPageIndex">1</entry>
<entry key="isSpawnUrl">true</entry>
<entry key="endInPageIndex">2</entry>
<entry key="retryTimes">5</entry>
<entry key="timeOut">10000</entry>
<entry key="sleepTimeThread">5000</entry>
<entry key="cycleRetryTimes">5</entry>
<entry key="userAgent">Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36</entry>
<entry key="sleepTimeSpider">10000</entry>
</properties>

15
crawler/dailyScheduledCrawler/fetch_networks/sql/detail_html.sql
View File

@ -1,15 +0,0 @@
(
`id` int(11) NOT NULL AUTO_INCREMENT ,
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
`crawledTime` datetime NULL DEFAULT NULL ,
`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
PRIMARY KEY (`id`)
)
ENGINE=InnoDB
DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
AUTO_INCREMENT=1
ROW_FORMAT=COMPACT
;

15
crawler/dailyScheduledCrawler/fetch_networks/sql/error_page.sql
View File

@ -1,15 +0,0 @@
(
`id` int(11) NOT NULL AUTO_INCREMENT ,
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
`crawledTime` datetime NULL DEFAULT NULL ,
`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
PRIMARY KEY (`id`)
)
ENGINE=InnoDB
DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
AUTO_INCREMENT=8388
ROW_FORMAT=COMPACT
;

17
crawler/dailyScheduledCrawler/fetch_networks/sql/list_html.sql
View File

@ -1,17 +0,0 @@
(
`id` int(11) NOT NULL AUTO_INCREMENT ,
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
`crawledTime` datetime NULL DEFAULT NULL ,
`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
`extracted` bigint(20) NULL DEFAULT NULL ,
PRIMARY KEY (`id`),
INDEX `pagemd5` (`pageMd5`) USING BTREE
)
ENGINE=InnoDB
DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
AUTO_INCREMENT=25
ROW_FORMAT=COMPACT
;

Some files were not shown because too many files have changed in this diff Show More