Compare commits
No commits in common. "master" and "develop" have entirely different histories.
|
|
@ -25,7 +25,6 @@
|
||||||
/new_osseanextractor/cursor
|
/new_osseanextractor/cursor
|
||||||
/new_osseanextractor/src/test/java/net/trustie/modeltest
|
/new_osseanextractor/src/test/java/net/trustie/modeltest
|
||||||
/new_osseanextractor.DS_Store
|
/new_osseanextractor.DS_Store
|
||||||
/new_osseanextractor/tasks.txt
|
|
||||||
|
|
||||||
/ow2_match/target/
|
/ow2_match/target/
|
||||||
/ow2_match/.project
|
/ow2_match/.project
|
||||||
|
|
@ -72,8 +71,10 @@
|
||||||
/trustie2/files/*
|
/trustie2/files/*
|
||||||
/trustie2/log/*
|
/trustie2/log/*
|
||||||
/trustie2/tmp/*
|
/trustie2/tmp/*
|
||||||
|
/trustie2/public/images/avatars/*
|
||||||
/trustie2/Gemfile.lock
|
/trustie2/Gemfile.lock
|
||||||
/trustie2/db/schema.rb
|
/trustie2/db/schema.rb
|
||||||
|
/trustie2/config/puma.rb
|
||||||
!/trustie2/solr/conf
|
!/trustie2/solr/conf
|
||||||
!/trustie2/solr/solr.xml
|
!/trustie2/solr/solr.xml
|
||||||
/trustie2/solr/*
|
/trustie2/solr/*
|
||||||
|
|
@ -108,12 +109,4 @@
|
||||||
/gather_program/target/
|
/gather_program/target/
|
||||||
/gather_program/log/*
|
/gather_program/log/*
|
||||||
/gather_program/.settings/
|
/gather_program/.settings/
|
||||||
/gather_program/.classpath
|
/gather_program/.classpath
|
||||||
|
|
||||||
|
|
||||||
/crawler/dailyScheduledCrawler/fetch_networks/target/
|
|
||||||
/crawler/dailyScheduledCrawler/fetch_networks/log/*
|
|
||||||
/crawler/daily_scheduler/log/*
|
|
||||||
/crawler/daily_scheduler/log_mem/*
|
|
||||||
/crawler/moreSmarterCrawler/fetch_networks/target/
|
|
||||||
/crawler/moreSmarterCrawler/fetch_networks/log/*
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<projectDescription>
|
|
||||||
<name>fetchnetworks</name>
|
|
||||||
<comment></comment>
|
|
||||||
<projects>
|
|
||||||
</projects>
|
|
||||||
<buildSpec>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.maven.ide.eclipse.maven2Builder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
<buildCommand>
|
|
||||||
<name>org.eclipse.m2e.core.maven2Builder</name>
|
|
||||||
<arguments>
|
|
||||||
</arguments>
|
|
||||||
</buildCommand>
|
|
||||||
</buildSpec>
|
|
||||||
<natures>
|
|
||||||
<nature>org.maven.ide.eclipse.maven2Nature</nature>
|
|
||||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
|
||||||
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
|
||||||
</natures>
|
|
||||||
</projectDescription>
|
|
||||||
|
|
@ -1,5 +0,0 @@
|
||||||
eclipse.preferences.version=1
|
|
||||||
encoding//src/main/java=UTF-8
|
|
||||||
encoding//src/main/resources=UTF-8
|
|
||||||
encoding//src/test/java=UTF-8
|
|
||||||
encoding/<project>=UTF-8
|
|
||||||
|
|
@ -1,2 +0,0 @@
|
||||||
eclipse.preferences.version=1
|
|
||||||
line.separator=\n
|
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
#Sun Jan 04 15:44:05 CST 2015
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
|
||||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
|
||||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
|
||||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
|
||||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
|
||||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
|
||||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
|
||||||
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
|
||||||
org.eclipse.jdt.core.compiler.source=1.7
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
activeProfiles=
|
|
||||||
eclipse.preferences.version=1
|
|
||||||
resolveWorkspaceProjects=true
|
|
||||||
version=1
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='51cto_blog'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='apache'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='bytes'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='cnblogs_news'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='cnblogs_q_solved'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='cnblogs_q_unsolved'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='code_project'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='csdn_ask'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='csdn_blog'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='csdn_topic'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx256m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='dewen_question'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='freecode'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='gna'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='ibm_post'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='ibm_project'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='iteye_ask'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='iteye_blog'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='lagou'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='linuxtone'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='lupaworld'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='neitui'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='openhub'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='oschina_project'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='oschina_question'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='phpchina'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
|
|
||||||
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
|
|
||||||
|
|
||||||
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
|
|
||||||
<layout class="org.apache.log4j.PatternLayout">
|
|
||||||
<param name="threshold" value="ERROR" />
|
|
||||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
|
||||||
</layout>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
|
|
||||||
<param name="File" value="./log/error.log" />
|
|
||||||
<param name="threshold" value="ERROR" />
|
|
||||||
<layout class="org.apache.log4j.PatternLayout">
|
|
||||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
|
||||||
</layout>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<appender name="file_log" class="org.apache.log4j.DailyRollingFileAppender">
|
|
||||||
<param name="File" value="./log/webmagic.log" />
|
|
||||||
<layout class="org.apache.log4j.PatternLayout">
|
|
||||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
|
||||||
</layout>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- 邮件（只有ERROR时才会发送！） -->
|
|
||||||
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
|
|
||||||
<param name="threshold" value="debug" />
|
|
||||||
<!-- 日志的错误级别 <param name="threshold" value="fatal"/> -->
|
|
||||||
<!-- 缓存文件大小，日志达到512K时发送Email -->
|
|
||||||
<param name="BufferSize" value="1" />
|
|
||||||
<!-- 单位K -->
|
|
||||||
<param name="From" value="ossean_debug@163.com" />
|
|
||||||
<param name="SMTPHost" value="smtp.163.com" />
|
|
||||||
<param name="Subject" value="ossean-crawler-debug-log4jMessage" />
|
|
||||||
<param name="To" value="gcm3651@126.com" />
|
|
||||||
<param name="SMTPUsername" value="ossean_debug" />
|
|
||||||
<param name="SMTPPassword" value="goodwell123" />
|
|
||||||
<layout class="org.apache.log4j.PatternLayout">
|
|
||||||
<param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss.SSS} [%p]-[%c] %m%n" />
|
|
||||||
</layout>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<!-- 数据库状态 -->
|
|
||||||
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
|
|
||||||
<param name="URL"
|
|
||||||
value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8" />
|
|
||||||
<param name="driver" value="com.mysql.jdbc.Driver" />
|
|
||||||
<param name="user" value="root" />
|
|
||||||
<param name="password" value="1234" />
|
|
||||||
<param name="sql"
|
|
||||||
value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%p','%c','%m','%l')" />
|
|
||||||
<!-- <layout class="org.apache.log4j.PatternLayout"> <param name="ConversionPattern"
|
|
||||||
value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES
|
|
||||||
('%d{yyyy-MM-dd HH:mm:ss}','%t','%.50p','%.50c','%.1000m','%.50l')" /> </layout> -->
|
|
||||||
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都 FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
|
|
||||||
<filter class="org.apache.log4j.varia.LevelRangeFilter">
|
|
||||||
<param name="LevelMin" value="DEBUG" />
|
|
||||||
<param name="LevelMax" value="FATAL" />
|
|
||||||
</filter>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<logger name="org.apache" additivity="false">
|
|
||||||
<level value="warn" />
|
|
||||||
<appender-ref ref="stdout" />
|
|
||||||
</logger>
|
|
||||||
|
|
||||||
<root>
|
|
||||||
<level value="info" />
|
|
||||||
<appender-ref ref="stdout" />
|
|
||||||
<!-- <appender-ref ref="file" /> -->
|
|
||||||
<!-- <appender-ref ref="file_log" /> -->
|
|
||||||
<!-- <appender-ref ref="MAIL" /> -->
|
|
||||||
<!-- <appender-ref ref="DATABASE" /> -->
|
|
||||||
</root>
|
|
||||||
|
|
||||||
</log4j:configuration>
|
|
||||||
|
|
@ -1,32 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://www.springframework.org/schema/beans
|
|
||||||
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
|
|
||||||
|
|
||||||
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
|
|
||||||
<property name="dataSource" ref="dataSource" />
|
|
||||||
</bean>
|
|
||||||
|
|
||||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
|
||||||
<property name="basePackage" value="net.trustie.webmagic" />
|
|
||||||
</bean>
|
|
||||||
|
|
||||||
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
|
|
||||||
destroy-method="close">
|
|
||||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
|
||||||
<property name="url" value="jdbc:mysql://192.168.80.104:3306/pages?characterEncoding=UTF-8" />
|
|
||||||
<property name="username" value="influx" />
|
|
||||||
<property name="password" value="influx1234" />
|
|
||||||
<property name="timeBetweenEvictionRunsMillis">
|
|
||||||
<value>21600000</value><!--6 hours-->
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property name="minEvictableIdleTimeMillis">
|
|
||||||
<value>21600000</value><!--connection的空闲时间大于这个值,就直接被关闭,并从连接池中删除-->
|
|
||||||
</property>
|
|
||||||
</bean>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</beans>
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
|
||||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xmlns:context="http://www.springframework.org/schema/context"
|
|
||||||
xmlns:mvc="http://www.springframework.org/schema/mvc"
|
|
||||||
xsi:schemaLocation="http://www.springframework.org/schema/mvc
|
|
||||||
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
|
|
||||||
http://www.springframework.org/schema/beans
|
|
||||||
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
|
|
||||||
http://www.springframework.org/schema/context
|
|
||||||
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
|
|
||||||
<context:annotation-config/>
|
|
||||||
|
|
||||||
<context:component-scan base-package="net.trustie.webmagic"/>
|
|
||||||
|
|
||||||
</beans>
|
|
||||||
|
|
@ -1,43 +0,0 @@
|
||||||
<!-- 输出通道"DATABASE"，输出方式是：输出所有级别的LOG到数据库 -->
|
|
||||||
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
|
|
||||||
<!--数据库的驱动，这里用的是MSSQL-->
|
|
||||||
<param name="driver" value="com.microsoft.jdbc.sqlserver.SQLServerDriver"/>
|
|
||||||
<!--这是Oracle用的驱动，在这里用MSSQL所以屏蔽掉了
|
|
||||||
<param name="driver" value="oracle.jdbc.driver.OracleDriver" />
|
|
||||||
-->
|
|
||||||
<!--要连接的数据库-->
|
|
||||||
<param name="URL" value="jdbc:microsoft:sqlserver://192.168.0.120:
|
|
||||||
|
|
||||||
1433;DatabaseName=test" />
|
|
||||||
<!--连接数据库的用户名-->
|
|
||||||
<param name="user" value="sa" />
|
|
||||||
<!--连接数据库的密码-->
|
|
||||||
<param name="password" value="sa" />
|
|
||||||
<!--向MSSQL数据库表LOG中插入数据的sql语句-->
|
|
||||||
<param name="sql" value=" INSERT INTO LOG
|
|
||||||
|
|
||||||
(LOGDATE,LOGLEVEL,LOGCLASS,LOGLOGGER,LOGMESSAGE)
|
|
||||||
|
|
||||||
values ('%d{yyyy-MM-dd HH:mm:ss}','%.50p', '%.50c',
|
|
||||||
|
|
||||||
'%.50l', '%.1000m')"/>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<!-- Oracle 的 insert 语句
|
|
||||||
<param name="sql" value=" insert into logrecord
|
|
||||||
|
|
||||||
(id,packageid,userid,syscodeid,info,logtime,loglevel)
|
|
||||||
|
|
||||||
values(?,?,?,?,?,to_date('%d{yyyy-MM-dd HH:mm:ss}',
|
|
||||||
|
|
||||||
'yyyy-MM-dd HH24:mi:ss'),'%p')" />
|
|
||||||
-->
|
|
||||||
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都
|
|
||||||
|
|
||||||
FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
|
|
||||||
<filter class="org.apache.log4j.varia.LevelRangeFilter">
|
|
||||||
<param name="LevelMin" value="ERROR"/>
|
|
||||||
<param name="LevelMax" value="FATAL"/>
|
|
||||||
</filter>
|
|
||||||
</appender>
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='freecode'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='slashdot'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='softpedia'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
SITE='sourceforge'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
|
|
||||||
SITE='stackoverflow'
|
|
||||||
|
|
||||||
find ./target/classes -name "*.properties"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.xml"|xargs rm -f
|
|
||||||
find ./target/classes -name "*.dic"|xargs rm -f
|
|
||||||
find ./target/classes/spring |xargs rm -f -r
|
|
||||||
find ./target/classes/sites |xargs rm -f -r
|
|
||||||
|
|
||||||
tmp='./bin/resources'
|
|
||||||
tmp='./sites':$tmp
|
|
||||||
tmp='./target/classes':$tmp
|
|
||||||
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
|
||||||
|
|
||||||
CLASSPATH=$tmp:$CLASSPATH
|
|
||||||
|
|
||||||
|
|
||||||
echo $CLASSPATH
|
|
||||||
JVM_ARGS="-Xmn48m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
|
||||||
#-XX:MinPermSize=128m"
|
|
||||||
#echo JVM_ARGS=$JVM_ARGS
|
|
||||||
#ulimit -n 400000
|
|
||||||
#echo "" > nohup.out
|
|
||||||
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
|
|
||||||
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
|
|
||||||
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
sh bin/cnblogs_news.sh
|
|
||||||
sh bin/cnblogs_q_solved.sh
|
|
||||||
sh bin/cnblogs_q_unsolved.sh
|
|
||||||
sh bin/csdn_ask.sh
|
|
||||||
sh bin/csdn_blog.sh
|
|
||||||
sh bin/csdn_topic.sh
|
|
||||||
sh bin/dewen_question.sh
|
|
||||||
sh bin/iteye_ask.sh
|
|
||||||
sh bin/openhub.sh
|
|
||||||
sh bin/oschina_project.sh
|
|
||||||
sh bin/oschina_question.sh
|
|
||||||
sh bin/sourceforge.sh
|
|
||||||
sh bin/stackoverflow.sh
|
|
||||||
Binary file not shown.
|
|
@ -1,25 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<entry key="domain">one</entry>
|
|
||||||
<entry key="threadNum">5</entry>
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<entry key="endInPageIndex">20000</entry>
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<entry key="timeOut">30000</entry>
|
|
||||||
<entry key="sleepTimeThread">5000</entry>
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<entry key="sleepTimeSpider">10000</entry>
|
|
||||||
<entry key="startUrl">http://www.oschina.net/question/</entry>
|
|
||||||
<entry key="urlList">http://www\.oschina\.net/question\?catalog=[12]\&show=active\&p=\d+</entry>
|
|
||||||
<entry key="urlPost">http://www\.oschina\.net/question/\d+_\d+</entry>
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<entry key="postTableName">html_test</entry>
|
|
||||||
<entry key="listTableName">target_url</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,202 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
||||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
||||||
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
|
|
||||||
<groupId>net.trustie</groupId>
|
|
||||||
<artifactId>fetchnetworks</artifactId>
|
|
||||||
<version>1.0-SNAPSHOT</version>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
||||||
<!-- m2eclipse wtp 0.12+ enabled to configure contextRoot, add by w.vela -->
|
|
||||||
<m2eclipse.wtp.contextRoot>/</m2eclipse.wtp.contextRoot>
|
|
||||||
<spring-version>3.1.1.RELEASE</spring-version>
|
|
||||||
<spring-security-version>3.1.0.RELEASE</spring-security-version>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
<build>
|
|
||||||
<sourceDirectory>${basedir}/src/main/java</sourceDirectory>
|
|
||||||
|
|
||||||
<plugins>
|
|
||||||
<plugin>
|
|
||||||
<groupId>org.apache.maven.plugins</groupId>
|
|
||||||
<artifactId>maven-compiler-plugin</artifactId>
|
|
||||||
<configuration>
|
|
||||||
<source>1.7</source>
|
|
||||||
<target>1.7</target>
|
|
||||||
</configuration>
|
|
||||||
</plugin>
|
|
||||||
<plugin>
|
|
||||||
<artifactId>maven-assembly-plugin</artifactId>
|
|
||||||
<version>2.5.1</version>
|
|
||||||
<configuration>
|
|
||||||
<descriptors>
|
|
||||||
<descriptor>src/main/assembly/assembly.xml</descriptor>
|
|
||||||
</descriptors>
|
|
||||||
</configuration>
|
|
||||||
</plugin>
|
|
||||||
</plugins>
|
|
||||||
</build>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
<dependency>
|
|
||||||
<groupId>xerces</groupId>
|
|
||||||
<artifactId>xerces</artifactId>
|
|
||||||
<version>2.4.0</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>dom4j</groupId>
|
|
||||||
<artifactId>dom4j</artifactId>
|
|
||||||
<version>1.6.1</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.json</groupId>
|
|
||||||
<artifactId>json</artifactId>
|
|
||||||
<version>20141113</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.google.guava</groupId>
|
|
||||||
<artifactId>guava</artifactId>
|
|
||||||
<version>14.0</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>us.codecraft</groupId>
|
|
||||||
<artifactId>xsoup</artifactId>
|
|
||||||
<version>0.2.4</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.github.dreamhead</groupId>
|
|
||||||
<artifactId>moco-core</artifactId>
|
|
||||||
<version>0.10.0</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.slf4j</groupId>
|
|
||||||
<artifactId>slf4j-log4j12</artifactId>
|
|
||||||
<version>1.7.7</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>commons-collections</groupId>
|
|
||||||
<artifactId>commons-collections</artifactId>
|
|
||||||
<version>3.2.1</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.assertj</groupId>
|
|
||||||
<artifactId>assertj-core</artifactId>
|
|
||||||
<version>1.7.0</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.commons</groupId>
|
|
||||||
<artifactId>commons-io</artifactId>
|
|
||||||
<version>1.3.2</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.jayway.jsonpath</groupId>
|
|
||||||
<artifactId>json-path</artifactId>
|
|
||||||
<version>0.8.1</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.alibaba</groupId>
|
|
||||||
<artifactId>fastjson</artifactId>
|
|
||||||
<version>1.2.3</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.jsoup</groupId>
|
|
||||||
<artifactId>jsoup</artifactId>
|
|
||||||
<version>1.7.2</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.httpcomponents</groupId>
|
|
||||||
<artifactId>httpclient</artifactId>
|
|
||||||
<version>4.3.4</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework</groupId>
|
|
||||||
<artifactId>spring-jdbc</artifactId>
|
|
||||||
<version>${spring-version}</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.commons</groupId>
|
|
||||||
<artifactId>commons-lang3</artifactId>
|
|
||||||
<version>3.1</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>javax.servlet</groupId>
|
|
||||||
<artifactId>servlet-api</artifactId>
|
|
||||||
<version>2.5</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>mysql</groupId>
|
|
||||||
<artifactId>mysql-connector-java</artifactId>
|
|
||||||
<version>5.1.18</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>commons-dbcp</groupId>
|
|
||||||
<artifactId>commons-dbcp</artifactId>
|
|
||||||
<version>1.3</version>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>junit</groupId>
|
|
||||||
<artifactId>junit</artifactId>
|
|
||||||
<version>4.7</version>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.mybatis</groupId>
|
|
||||||
<artifactId>mybatis</artifactId>
|
|
||||||
<version>3.1.1</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.mybatis</groupId>
|
|
||||||
<artifactId>mybatis-spring</artifactId>
|
|
||||||
<version>1.1.1</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework</groupId>
|
|
||||||
<artifactId>spring-test</artifactId>
|
|
||||||
<version>${spring-version}</version>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>net.trustie</groupId>
|
|
||||||
<artifactId>webmagic-core-fix</artifactId>
|
|
||||||
<version>1.0</version>
|
|
||||||
<scope>system</scope>
|
|
||||||
<systemPath>${project.basedir}/lib/webmagic-core-fix.jar</systemPath>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>net.trustie</groupId>
|
|
||||||
<artifactId>webmagic-extension-fix</artifactId>
|
|
||||||
<version>1.0</version>
|
|
||||||
<scope>system</scope>
|
|
||||||
<systemPath>${project.basedir}/lib/webmagic-extension-fix.jar</systemPath>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.seleniumhq.selenium</groupId>
|
|
||||||
<artifactId>selenium-java</artifactId>
|
|
||||||
<version>2.42.2</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
|
|
||||||
</dependencies>
|
|
||||||
|
|
||||||
</project>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>51cto</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>div.rightbox>div.r_li>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://blog.51cto.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://blog.51cto.com/original/0/</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>16</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>51cto_blog</domain>
|
|
||||||
<minInterval>20000</minInterval>
|
|
||||||
<maxInterval>30000</maxInterval>
|
|
||||||
<minlongSleepTime>600000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>1200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,34 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">csdn_topic</entry>
|
|
||||||
<!-- 有序爬取中，起始页页码-->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- 有序爬取中，结束页页码-->
|
|
||||||
<entry key="endInPageIndex">10</entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
|
|
||||||
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">50</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">20000</entry>
|
|
||||||
<!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>1</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>td.body>div>div>ul>li> a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://projects.apache.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://projects.apache.org/indexes/alpha.html</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>864000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>apache</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>30000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">51cto_blog</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">30</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://blog.51cto.com/original/0/1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.rightbox>div.r_li>h4>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://blog.51cto.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://blog.51cto.com/original/0/</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">16</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">apache</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">30</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://projects.apache.org/indexes/alpha.html</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">table>tbody>tr>td.body>div>div>ul>li> a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://projects.apache.org/indexes/alpha.html</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://projects.apache.org/indexes/alpha.html</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">1</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">8640000</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">cnblogs_news</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">4845</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://news.cnblogs.com/n/page/1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div#news_list>div.news_block>div.content>h2.news_entry>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://news.cnblogs.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://news.cnblogs.com/n/page/</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">5</entry>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">cnblogs_q_solved</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">160</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">4000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://q.cnblogs.com/list/solved?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://q.cnblogs.com/list/solved?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">2</entry>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,72 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">cnblogs_q_unsolved</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">160</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">4000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://q.cnblogs.com/list/unsolved?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://q.cnblogs.com/list/unsolved?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">2</entry>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">codeproject</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">21</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.content-list-item>div.title>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://www.codeproject.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">1</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">dewen_question</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">1114</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://www.dewen.io/questions?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://www.dewen.io</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://www.dewen.io/questions?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">1</entry>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">432000</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">gna</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">1348</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=1&max_rows=1#results</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">table>tbody>tr>td>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://gna.org</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">&max_rows=1#results</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">200</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">432000</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">lupaworld</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">2</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">479</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://www.lupaworld.com/forum-13746-2.html</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">table#threadlisttableid>tbody>tr>th>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://www.lupaworld.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://www.lupaworld.com/forum-13746-</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">.html</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">2</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">2592000</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="domain">openhub</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">66784</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">60000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">https://www.openhub.net/p?page=1&q=&sort=users</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.project>h2.title>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">60000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">https://www.openhub.net</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">https://www.openhub.net/p?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">&q=&sort=users</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">500</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">100000</entry>
|
|
||||||
|
|
||||||
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">phpchina</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">1289</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://bbs.phpchina.com/forum-17-1.html</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">table#threadlisttableid>tbody>tr>th>a.s.xst</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://bbs.phpchina.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://bbs.phpchina.com/forum-17-</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">.html</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">2</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">1296000</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">softpedia</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">30</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://linux.softpedia.com/index1.shtml</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div#sjmp>div>h4>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://linux.softpedia.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://linux.softpedia.com/index</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">.shtml</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">10</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="domain">sourceforge</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">17621</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">60000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">10000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://sourceforge.net/directory/?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">ul.projects>li>div.project_info>header>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://sourceforge.net</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://sourceforge.net/directory/?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">500</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">250000</entry>
|
|
||||||
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- 有序爬取时，每次生成页面链接数目-->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">stackoverflow</entry>
|
|
||||||
<!-- 线程数目-->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- 有序爬取中，起始页页码-->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- 有序爬取中，结束页页码-->
|
|
||||||
<entry key="endInPageIndex">25000</entry>
|
|
||||||
<!-- 页面下载超时，重试次数-->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- 下载时，连接超时等待时间-->
|
|
||||||
<entry key="timeOut">60000</entry>
|
|
||||||
<!-- 超时时间，单位是毫秒-->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- 循环重试次数-->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- 请求头的UserAgent信息-->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
|
|
||||||
<entry key="sleepTimeSpider">10000</entry>
|
|
||||||
<!-- 起始种子URL -->
|
|
||||||
<entry key="startUrl">http://stackoverflow.com/questions?pagesize=50&sort=newest&page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div#mainbar>div#questions>div.question-summary>div.summary>h3>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://stackoverflow.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://stackoverflow.com/questions?pagesize=50&sort=newest&page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">250</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">20000</entry>
|
|
||||||
|
|
||||||
<entry key="minInterval">2000</entry>
|
|
||||||
<entry key="maxInterval">3000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">2400000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">7200000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">14400000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">linuxtone</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">64</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://bbs.linuxtone.org/forum-15-1.html</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">table>tbody>tr>th>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://bbs.linuxtone.org</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://bbs.linuxtone.org/forum-15-</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">.html</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">2</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">2592000</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">iteye_blog</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">15</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://www.iteye.com/blogs?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.content>h3>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://www.iteye.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://www.iteye.com/blogs?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">2</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>question</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>2000</endInPageIndex>
|
|
||||||
<urlPost>table#threadslist>tbody>tr>td>div>div.threadbit_title>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bytes.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bytes.com/answers/</prefixUrl>
|
|
||||||
<postfixUrl>/</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>bytes</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,32 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>4845</endInPageIndex>
|
|
||||||
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://news.cnblogs.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://news.cnblogs.com/n/page/</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>5</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>cnblogs_news</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>160</endInPageIndex>
|
|
||||||
<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://q.cnblogs.com/list/solved?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>cnblogs_q_solved</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://q.cnblogs.com/list/unsolved?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>cnblogs_q_unsolved</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>259</endInPageIndex>
|
|
||||||
<urlPost>div.content-list-item>div.title>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.codeproject.com</fixAllRelativeHrefs>
|
|
||||||
<!--<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&la_app=20&la_minscore=3.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=</prefixUrl>-->
|
|
||||||
<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=366&la_app=20&la_minscore=1.0&la_allattr=False&at=1%2c3%2c6%2c8&pgnum=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>codeproject</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>2496</endInPageIndex>
|
|
||||||
<urlPost>div.common_con>div.questions_detail_con>dl>dt>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://ask.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://ask.csdn.net/?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>25</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>csdn_ask</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>60000</maxInterval>
|
|
||||||
<minlongSleepTime>7200000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>10800000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>101</endInPageIndex>
|
|
||||||
<urlPost>div.main_center>div.blog_list>h1>a:not(.category)</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://blog.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://blog.csdn.net/index.html?&page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>3</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>csdn_blog</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>60000</maxInterval>
|
|
||||||
<minlongSleepTime>7200000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>10800000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,81 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>mobile</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>808</endInPageIndex>
|
|
||||||
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.csdn.net/forums/Mobile?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>8</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>CloudComputing</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>22</endInPageIndex>
|
|
||||||
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.csdn.net/forums/CloudComputing?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>java</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>2555</endInPageIndex>
|
|
||||||
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.csdn.net/forums/Java?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>4</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>web</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>1853</endInPageIndex>
|
|
||||||
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.csdn.net/forums/WebDevelop?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>3</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>OtherLanguage</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>336</endInPageIndex>
|
|
||||||
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.csdn.net/forums/OtherLanguage?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>csdn_topic</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>60000</maxInterval>
|
|
||||||
<minlongSleepTime>7200000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>10800000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>1114</endInPageIndex>
|
|
||||||
<urlPost>div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.dewen.io</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.dewen.io/questions?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>432000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>dewen_question</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="domain">freecode</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">4799</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">30000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">10000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://www.freecode.com/projects?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.project>h2.release-head>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">60000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://www.freecode.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://www.freecode.com/projects?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">5</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">20000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>1348</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://gna.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://gna.org/search/?type_of_search=soft&words=%2A&type=1&offset=</prefixUrl>
|
|
||||||
<postfixUrl>&max_rows=1#results</postfixUrl>
|
|
||||||
<incrementalPages>100</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>gna</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>blog</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>138</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.ibm.com/developerworks/cn/views/opensource/libraryview.jsp?site_id=10&contentarea_by=%E6%89%80%E6%9C%89%E4%B8%93%E5%8C%BA&sort_by=&sort_order=2&start=</prefixUrl>
|
|
||||||
<postfixUrl>00&end=13874&topic_by=-1&product_by=&type_by=%E6%89%80%E6%9C%89%E7%B1%BB%E5%88%AB&show_abstract=false&search_by=&industry_by=&series_title_by=</postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>6048000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>ibm_post</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>1</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.ibm.com/developerworks/views/opensource/projects.jsp</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>864000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>ibm_project</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="domain">iteye_ask</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">5329</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">30000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://www.iteye.com/ask?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div#ask_list>div.question-summary>div.summary>h3>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">60000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://www.iteye.com/</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://www.iteye.com/ask?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">50</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">20000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,78 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name>mobile</name>
|
|
||||||
<startPageIndex>z</startPageIndex>
|
|
||||||
<endInPageIndex>111</endInPageIndex>
|
|
||||||
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.iteye.com/forums/board/mobile?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>16</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<name>language</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>242</endInPageIndex>
|
|
||||||
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.iteye.com/forums/board/language?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<name>web</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>320</endInPageIndex>
|
|
||||||
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.iteye.com/forums/board/web?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<name>java</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>754</endInPageIndex>
|
|
||||||
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.iteye.com/forums/board/Java?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<name>tech</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>242</endInPageIndex>
|
|
||||||
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.iteye.com/forums/board/Tech?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>iteye_bbs</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>50000</maxInterval>
|
|
||||||
<minlongSleepTime>3600000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name>blog</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>15</endInPageIndex>
|
|
||||||
<urlPost>div.content>h3>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.iteye.com/blogs?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>iteye_blog</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>50000</maxInterval>
|
|
||||||
<minlongSleepTime>3600000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">javaforge</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">31</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">5000</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://javaforge.com/listProjects.spr?page=1&displayAllProjects=on</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">table#entry>tbody>tr>td:nth-child(1) > a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://javaforge.com</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://javaforge.com/listProjects.spr?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl">&displayAllProjects=on</entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">10</entry>
|
|
||||||
|
|
||||||
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">86400</entry>
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>houduan</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost> #container > div.content>ul> li> div> div > a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.lagou.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.lagou.com/jobs/list_%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91?kd=%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91&spc=1&pl=&gj=&xl=&yx=&gx=&st=&labelWords=label&lc=&workAddress=&city=%E5%85%A8%E5%9B%BD&requestId=&pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>16</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>lagou</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,81 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>System</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>64</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>th>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.linuxtone.org/forum-15-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>2592000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>server</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>th>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.linuxtone.org/forum-22-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>2592000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>dbserver</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>32</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>th>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.linuxtone.org/forum-24-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>2592000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>yngz</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>32</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>th>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.linuxtone.org/forum-49-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>2592000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>fzjh</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>24</endInPageIndex>
|
|
||||||
<urlPost>table>tbody>tr>th>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.linuxtone.org/forum-26-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>2592000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>linuxtone</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>479</endInPageIndex>
|
|
||||||
<urlPost>table#threadlisttableid>tbody>tr>th>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.lupaworld.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.lupaworld.com/forum-13746-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>2592000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>lupaworld</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>610</endInPageIndex>
|
|
||||||
<urlPost> ul>li>div.cont>div.jobnote.clearfix > div.jobnote-l>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.neitui.me</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.neitui.me/neitui/type=all&page=</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>16</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>neitui</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,152 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>dev</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>370</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/1?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>language</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>652</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/36?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>website</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>41</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/57?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>server</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>145</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/75?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>database</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>102</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/90?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>plugin</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>53</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/52?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>OS</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>42</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/155?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>manage</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>12</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/319?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>dev_manage</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>9</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/159?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>util</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>7</endInPageIndex>
|
|
||||||
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.open-open.com/lib/list/322?pn=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>open_open</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>66784</endInPageIndex>
|
|
||||||
<urlPost>div.well>h2.title>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>https://www.openhub.net</fixAllRelativeHrefs>
|
|
||||||
<!--<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
|
|
||||||
<postfixUrl>&q=&sort=users</postfixUrl>
|
|
||||||
<incrementalPages>300</incrementalPages>
|
|
||||||
<incrementSleepTime>300000</incrementSleepTime> -->
|
|
||||||
|
|
||||||
<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
|
|
||||||
<postfixUrl>&q=&sort=new</postfixUrl>
|
|
||||||
<incrementalPages>8</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>openhub</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>2044</endInPageIndex>
|
|
||||||
<urlPost>div.ProjectList>ul.List>li>h3>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.oschina.net/project/list?prefix=&sort=time&p=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>oschina_project</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>60000</maxInterval>
|
|
||||||
<minlongSleepTime>7200000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>10800000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>2361</endInPageIndex>
|
|
||||||
<urlPost>ul.list>li.question>div.question-detail>strong>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://www.oschina.net/question?catalog=1&show=active&p=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>oschina_question</domain>
|
|
||||||
<minInterval>30000</minInterval>
|
|
||||||
<maxInterval>60000</maxInterval>
|
|
||||||
<minlongSleepTime>7200000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>10800000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>1800000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>3600000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="pageF">3</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="domain">ossean</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="threadNum">3</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="endInPageIndex">5777</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="sleepTimeSpider">500</entry>
|
|
||||||
<!-- 起始页 -->
|
|
||||||
<entry key="startUrl">http://localhost/open_source_projects?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">ul.projects>li.project-table>div.root>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://localhost</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://localhost/open_source_projects?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">5</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">0</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,70 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>ask</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>1289</endInPageIndex>
|
|
||||||
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.phpchina.com/forum-17-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>server_os</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>91</endInPageIndex>
|
|
||||||
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.phpchina.com/forum-195-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>web</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>81</endInPageIndex>
|
|
||||||
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.phpchina.com/forum-213-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>db</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>43</endInPageIndex>
|
|
||||||
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://bbs.phpchina.com/forum-196-</prefixUrl>
|
|
||||||
<postfixUrl>.html</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>1296000</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>phpchina</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,72 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<!-- 有序爬取时，每次生成页面链接数目-->
|
|
||||||
<entry key="pageF">3</entry>
|
|
||||||
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
|
|
||||||
<entry key="domain">csdn_topic</entry>
|
|
||||||
<!-- 线程数目-->
|
|
||||||
<entry key="threadNum">3</entry>
|
|
||||||
<!-- 有序爬取中，起始页页码-->
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<!-- 有序爬取中，结束页页码-->
|
|
||||||
<entry key="endInPageIndex">10</entry>
|
|
||||||
<!-- 页面下载超时，重试次数-->
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<!-- 下载时，连接超时等待时间-->
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<!-- 单位是毫秒-->
|
|
||||||
<entry key="sleepTimeThread">0</entry>
|
|
||||||
<!-- 循环重试次数-->
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<!-- 请求头的UserAgent信息-->
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
|
|
||||||
<!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
|
|
||||||
<entry key="sleepTimeSpider">4000</entry>
|
|
||||||
<!-- 起始种子URL -->
|
|
||||||
<entry key="startUrl">http://bbs.csdn.net/tech_hot_topics?page=1</entry>
|
|
||||||
<!-- 列表页的规则 -->
|
|
||||||
<entry key="urlList"></entry>
|
|
||||||
<!-- 详情页链接抽取规则 -->
|
|
||||||
<entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="offset">0</entry>
|
|
||||||
<!-- -->
|
|
||||||
<entry key="limitLine">10</entry>
|
|
||||||
<!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
|
|
||||||
<entry key="noUrlWaitTime">10000</entry>
|
|
||||||
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
|
|
||||||
<entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
|
|
||||||
<!-- 列表页url规则的前缀 -->
|
|
||||||
<entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
|
|
||||||
<!-- 列表页url规则的后缀 -->
|
|
||||||
<entry key="postfixUrl"></entry>
|
|
||||||
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
|
|
||||||
<entry key="extractMethod">CSSPath</entry>
|
|
||||||
<!-- 代理ip -->
|
|
||||||
<entry key="proxyIp">192.168.245.1</entry>
|
|
||||||
<!-- 代理端口号 -->
|
|
||||||
<entry key="proxyPort">3128</entry>
|
|
||||||
<!-- 爬取模式 0:mirror 或者是 1:increment-->
|
|
||||||
<entry key="mode">0</entry>
|
|
||||||
<!-- 增量模式观察的页数 -->
|
|
||||||
<entry key="incrementalPages">50</entry>
|
|
||||||
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
|
|
||||||
<entry key="incrementSleepTime">20000</entry>
|
|
||||||
|
|
||||||
<!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
|
|
||||||
<entry key="minInterval">10000</entry>
|
|
||||||
<entry key="maxInterval">20000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠时间的上下限 -->
|
|
||||||
<entry key="minlongSleepTime">1800000</entry>
|
|
||||||
<entry key="MaxlongSleepTime">7200000</entry>
|
|
||||||
|
|
||||||
<!--长时间睡眠时候的睡眠间隔的上下限 -->
|
|
||||||
<entry key="minlongSleepInterval">3600000</entry>
|
|
||||||
<entry key="MaxlongSleepInterval">7200000</entry>
|
|
||||||
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,32 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>66</endInPageIndex>
|
|
||||||
|
|
||||||
<urlPost> #firehoselist>article>header>h2>span>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://slashdot.org</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://slashdot.org/?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>slashdot</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
||||||
|
|
@ -1,134 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>linux</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://linux.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://linux.softpedia.com/index</prefixUrl>
|
|
||||||
<postfixUrl>.shtml</postfixUrl>
|
|
||||||
<incrementalPages>3</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>win</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://win.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://win.softpedia.com/index</prefixUrl>
|
|
||||||
<postfixUrl>.shtml</postfixUrl>
|
|
||||||
<incrementalPages>3</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>mac</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>35</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://mac.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://mac.softpedia.com/index</prefixUrl>
|
|
||||||
<postfixUrl>.shtml</postfixUrl>
|
|
||||||
<incrementalPages>5</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>windows-phone</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>46</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://mobile.softpedia.com/windows-phone,</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>5</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>android</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>35</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://mobile.softpedia.com/android,</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>ios</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>22</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://mobile.softpedia.com/ios,</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>1</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>driver</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://drivers.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://drivers.softpedia.com/index</prefixUrl>
|
|
||||||
<postfixUrl>.shtml</postfixUrl>
|
|
||||||
<incrementalPages>10</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>game</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://games.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://games.softpedia.com/index</prefixUrl>
|
|
||||||
<postfixUrl>.shtml</postfixUrl>
|
|
||||||
<incrementalPages>2</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name>webscripts</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>30</endInPageIndex>
|
|
||||||
<urlPost>h4>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://webscripts.softpedia.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://webscripts.softpedia.com/index/latest-scripts-3</prefixUrl>
|
|
||||||
<postfixUrl>-0-0.html</postfixUrl>
|
|
||||||
<incrementalPages>3</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>softpedia</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>17621</endInPageIndex>
|
|
||||||
<urlPost>ul.projects>li>div.project_info>header>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://sourceforge.net</fixAllRelativeHrefs>
|
|
||||||
|
|
||||||
<!-- <prefixUrl>http://sourceforge.net/directory/?page=</prefixUrl>-->
|
|
||||||
|
|
||||||
<prefixUrl>http://sourceforge.net/directory/freshness%3Arecently-updated/?sort=update&page=</prefixUrl>
|
|
||||||
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<!--<incrementalPages>300</incrementalPages>
|
|
||||||
<incrementSleepTime>250000</incrementSleepTime>-->
|
|
||||||
<incrementalPages>8</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>sourceforge</domain>
|
|
||||||
<minInterval>10000</minInterval>
|
|
||||||
<maxInterval>20000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,66 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!-- <!DOCTYPE site[
|
|
||||||
<!ELEMENT site (subSites,properties)>
|
|
||||||
<!ELEMENT subSites (subSite+)>
|
|
||||||
<!ELEMENT properties (entry+)>
|
|
||||||
<!ELEMENT subSite (name,startPageIndex,endInPageIndex,urlPost,fixAllRelativeHrefs,prefixUrl,postfixUrl,incrementalPages,incrementSleepTime,mode)>
|
|
||||||
<!ELEMENT comment (#PCDATA)>
|
|
||||||
<!ELEMENT name (#PCDATA)>
|
|
||||||
<!ELEMENT starPageIndex (#PCDATA)>
|
|
||||||
<!ELEMENT endInPageIndex (#PCDATA)>
|
|
||||||
<!ELEMENT urlPost (#PCDATA)>
|
|
||||||
<!ELEMENT fixAllRelativeHrefs (#PCDATA)>
|
|
||||||
<!ELEMENT prefixUrl (#PCDATA)>
|
|
||||||
<!ELEMENT postfixUrl (#PCDATA)>
|
|
||||||
<!ELEMENT incrementalPages (#PCDATA)>
|
|
||||||
<!ELEMENT incrementSleepTime (#PCDATA)>
|
|
||||||
<!ELEMENT mode (#PCDATA)>
|
|
||||||
<!ELEMENT entry (#PCDATA)>
|
|
||||||
<!ATTLIST entry key CDATA "ID">
|
|
||||||
]> -->
|
|
||||||
|
|
||||||
<!-- <!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">-->
|
|
||||||
|
|
||||||
<site>
|
|
||||||
|
|
||||||
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<name>database</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>20</endInPageIndex>
|
|
||||||
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://localhost:8080/SpiderTest</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://localhost:8080/SpiderTest/database?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>15</incrementalPages>
|
|
||||||
<incrementSleepTime>300</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
<subSite>
|
|
||||||
<name>os</name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>20</endInPageIndex>
|
|
||||||
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://localhost:8080/SpiderTest</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://localhost:8080/SpiderTest/os?page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>15</incrementalPages>
|
|
||||||
<incrementSleepTime>300</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>SpiderTes</domain>
|
|
||||||
<minInterval>1000</minInterval>
|
|
||||||
<maxInterval>3000</maxInterval>
|
|
||||||
<minlongSleepTime>1800000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>7200000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>3600000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>7200000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<site>
|
|
||||||
<subSites>
|
|
||||||
<subSite>
|
|
||||||
<!-- 普通网站name不用写 -->
|
|
||||||
<name></name>
|
|
||||||
<startPageIndex>1</startPageIndex>
|
|
||||||
<endInPageIndex>25000</endInPageIndex>
|
|
||||||
<urlPost>div#mainbar>div#questions>div.question-summary>div.summary>h3>a</urlPost>
|
|
||||||
<fixAllRelativeHrefs>http://stackoverflow.com</fixAllRelativeHrefs>
|
|
||||||
<prefixUrl>http://stackoverflow.com/questions?pagesize=50&sort=newest&page=</prefixUrl>
|
|
||||||
<postfixUrl></postfixUrl>
|
|
||||||
<incrementalPages>250</incrementalPages>
|
|
||||||
<incrementSleepTime>86400</incrementSleepTime>
|
|
||||||
<mode>0</mode>
|
|
||||||
</subSite>
|
|
||||||
</subSites>
|
|
||||||
|
|
||||||
<properties>
|
|
||||||
<domain>stackoverflow</domain>
|
|
||||||
<minInterval>2000</minInterval>
|
|
||||||
<maxInterval>3000</maxInterval>
|
|
||||||
<minlongSleepTime>900000</minlongSleepTime>
|
|
||||||
<maxlongSleepTime>1800000</maxlongSleepTime>
|
|
||||||
<minlongSleepInterval>7200000</minlongSleepInterval>
|
|
||||||
<maxlongSleepInterval>14400000</maxlongSleepInterval>
|
|
||||||
</properties>
|
|
||||||
|
|
||||||
</site>
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
||||||
<properties>
|
|
||||||
<comment>Spider</comment>
|
|
||||||
<entry key="pageF">1</entry>
|
|
||||||
<entry key="domain">ques_oschina</entry>
|
|
||||||
<entry key="threadNum">2</entry>
|
|
||||||
<entry key="startPageIndex">1</entry>
|
|
||||||
<entry key="isSpawnUrl">true</entry>
|
|
||||||
<entry key="endInPageIndex">2</entry>
|
|
||||||
<entry key="retryTimes">5</entry>
|
|
||||||
<entry key="timeOut">10000</entry>
|
|
||||||
<entry key="sleepTimeThread">5000</entry>
|
|
||||||
<entry key="cycleRetryTimes">5</entry>
|
|
||||||
<entry key="userAgent">Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36</entry>
|
|
||||||
<entry key="sleepTimeSpider">10000</entry>
|
|
||||||
</properties>
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
(
|
|
||||||
`id` int(11) NOT NULL AUTO_INCREMENT ,
|
|
||||||
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
|
|
||||||
`crawledTime` datetime NULL DEFAULT NULL ,
|
|
||||||
`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
PRIMARY KEY (`id`)
|
|
||||||
)
|
|
||||||
ENGINE=InnoDB
|
|
||||||
DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
|
|
||||||
AUTO_INCREMENT=1
|
|
||||||
ROW_FORMAT=COMPACT
|
|
||||||
;
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
(
|
|
||||||
`id` int(11) NOT NULL AUTO_INCREMENT ,
|
|
||||||
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
|
|
||||||
`crawledTime` datetime NULL DEFAULT NULL ,
|
|
||||||
`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`type` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
PRIMARY KEY (`id`)
|
|
||||||
)
|
|
||||||
ENGINE=InnoDB
|
|
||||||
DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
|
|
||||||
AUTO_INCREMENT=8388
|
|
||||||
ROW_FORMAT=COMPACT
|
|
||||||
;
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
(
|
|
||||||
`id` int(11) NOT NULL AUTO_INCREMENT ,
|
|
||||||
`url` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`html` mediumtext CHARACTER SET utf8 COLLATE utf8_general_ci NULL ,
|
|
||||||
`crawledTime` datetime NULL DEFAULT NULL ,
|
|
||||||
`urlMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`pageMd5` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`history` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL ,
|
|
||||||
`extracted` bigint(20) NULL DEFAULT NULL ,
|
|
||||||
PRIMARY KEY (`id`),
|
|
||||||
INDEX `pagemd5` (`pageMd5`) USING BTREE
|
|
||||||
)
|
|
||||||
ENGINE=InnoDB
|
|
||||||
DEFAULT CHARACTER SET=utf8 COLLATE=utf8_general_ci
|
|
||||||
AUTO_INCREMENT=25
|
|
||||||
ROW_FORMAT=COMPACT
|
|
||||||
;
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue