zaihui
/
educoder
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

clean branch

This commit is contained in:
Fisher Yu 2017-10-16 14:49:45 +08:00
parent df8d208409
commit 10b5e36c46
1464 changed files with 0 additions and 1492639 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
crawler/.DS_Store vendored
View File

Binary file not shown.

6
crawler/.idea/compiler.xml
View File

@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<bytecodeTargetLevel target="1.8" />
</component>
</project>

9
crawler/.idea/crawler.iml
View File

@ -1,9 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

71
crawler/.idea/misc.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MavenImportPreferences">
<option name="generalSettings">
<MavenGeneralSettings>
<option name="mavenHome" value="/usr/local/Cellar/maven/3.5.0/libexec" />
</MavenGeneralSettings>
</option>
</component>
<component name="masterDetails">
<states>
<state key="GlobalLibrariesConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="JdkListConfigurable.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ProjectLibrariesConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ScopeChooserConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>

8
crawler/.idea/modules.xml
View File

@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/crawler.iml" filepath="$PROJECT_DIR$/.idea/crawler.iml" />
</modules>
</component>
</project>

251
crawler/.idea/workspace.xml
View File

@ -1,251 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="32a73eed-3716-4f88-8891-2b38051ca637" name="Default" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="GradleLocalSettings">
<option name="externalProjectsViewState">
<projects_view />
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="ProjectFrameBounds">
<option name="width" value="1280" />
<option name="height" value="800" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="crawler" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="crawler" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="moreSmarterCrawler" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="crawler" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="crawler" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
<pane id="Scratches" />
<pane id="Scope" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="File.Encoding" />
<property name="project.structure.last.edited" value="Project" />
<property name="project.structure.proportion" value="0.15" />
<property name="project.structure.side.proportion" value="0.0" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="FullScreen" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager">
<configuration default="true" type="#org.jetbrains.idea.devkit.run.PluginConfigurationType" factoryName="Plugin">
<module name="" />
<option name="VM_PARAMETERS" value="-Xmx512m -Xms256m -XX:MaxPermSize=250m -ea" />
<option name="PROGRAM_PARAMETERS" />
<predefined_log_file id="idea.log" enabled="true" />
<method />
</configuration>
<configuration default="true" type="Applet" factoryName="Applet">
<option name="HTML_USED" value="false" />
<option name="WIDTH" value="400" />
<option name="HEIGHT" value="300" />
<option name="POLICY_FILE" value="$APPLICATION_HOME_DIR$/bin/appletviewer.policy" />
<module />
<method />
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" />
<option name="VM_PARAMETERS" />
<option name="PROGRAM_PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="ENABLE_SWING_INSPECTOR" value="false" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<module name="" />
<envs />
<method />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<envs />
<patterns />
<method />
</configuration>
<configuration default="true" type="Remote" factoryName="Remote">
<option name="USE_SOCKET_TRANSPORT" value="true" />
<option name="SERVER_MODE" value="false" />
<option name="SHMEM_ADDRESS" value="javadebug" />
<option name="HOST" value="localhost" />
<option name="PORT" value="5005" />
<method />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<option name="OUTPUT_DIRECTORY" />
<option name="ANNOTATION_TYPE" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<envs />
<properties />
<listeners />
<method />
</configuration>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="32a73eed-3716-4f88-8891-2b38051ca637" name="Default" comment="" />
<created>1503302070409</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1503302070409</updated>
<workItem from="1503302072171" duration="53000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="53000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="0" width="1280" height="800" extended-state="0" />
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Nl-Palette" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Properties" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Theme Preview" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="processedProjectFiles" value="true" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
</project>

36
crawler/dailyScheduledCrawler/fetch_networks/.classpath
View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry exported="true" kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry including="**/*.java" kind="src" output="target/classes" path="src/main/resources"/>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" path="src/main/assembly"/>
<classpathentry kind="src" output="target/classes" path="sites">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/junit-4.11"/>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

29
crawler/dailyScheduledCrawler/fetch_networks/.project
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>fetchnetworks</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.maven.ide.eclipse.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.maven.ide.eclipse.maven2Nature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

5
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.resources.prefs
View File

@ -1,5 +0,0 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8

2
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.core.runtime.prefs
View File

@ -1,2 +0,0 @@
eclipse.preferences.version=1
line.separator=\n

14
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.jdt.core.prefs
View File

@ -1,14 +0,0 @@
#Sun Jan 04 15:44:05 CST 2015
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.7

4
crawler/dailyScheduledCrawler/fetch_networks/.settings/org.eclipse.m2e.core.prefs
View File

@ -1,4 +0,0 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

30
crawler/dailyScheduledCrawler/fetch_networks/bin/51cto_blog.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='51cto_blog'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/apache.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='apache'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/bytes.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='bytes'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_news.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='cnblogs_news'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_solved.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='cnblogs_q_solved'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/cnblogs_q_unsolved.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='cnblogs_q_unsolved'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/code_project.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='code_project'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_ask.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='csdn_ask'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_blog.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='csdn_blog'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/csdn_topic.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='csdn_topic'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx256m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/dewen_question.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='dewen_question'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/freecode.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='freecode'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/gna.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='gna'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_post.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='ibm_post'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/ibm_project.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='ibm_project'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_ask.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='iteye_ask'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/iteye_blog.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='iteye_blog'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/lagou.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='lagou'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/linuxtone.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='linuxtone'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/lupaworld.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='lupaworld'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/neitui.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='neitui'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/openhub.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='openhub'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_project.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='oschina_project'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/oschina_question.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='oschina_question'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/phpchina.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='phpchina'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

79
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/log4j.xml
View File

@ -1,79 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="threshold" value="ERROR" />
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="./log/error.log" />
<param name="threshold" value="ERROR" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<appender name="file_log" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="./log/webmagic.log" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<!-- 邮件（只有ERROR时才会发送！） -->
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
<param name="threshold" value="debug" />
<!-- 日志的错误级别 <param name="threshold" value="fatal"/> -->
<!-- 缓存文件大小，日志达到512K时发送Email -->
<param name="BufferSize" value="1" />
<!-- 单位K -->
<param name="From" value="ossean_debug@163.com" />
<param name="SMTPHost" value="smtp.163.com" />
<param name="Subject" value="ossean-crawler-debug-log4jMessage" />
<param name="To" value="gcm3651@126.com" />
<param name="SMTPUsername" value="ossean_debug" />
<param name="SMTPPassword" value="goodwell123" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%-d{yyyy-MM-dd HH:mm:ss.SSS} [%p]-[%c] %m%n" />
</layout>
</appender>
<!-- 数据库状态 -->
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
<param name="URL"
value="jdbc:mysql://127.0.0.1:3306/webmagic?characterEncoding=UTF-8" />
<param name="driver" value="com.mysql.jdbc.Driver" />
<param name="user" value="root" />
<param name="password" value="1234" />
<param name="sql"
value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES ('%d{yyyy-MM-dd HH:mm:ss}','%t','%p','%c','%m','%l')" />
<!-- <layout class="org.apache.log4j.PatternLayout"> <param name="ConversionPattern"
value="INSERT INTO log4j(stamp,thread,info_level,class,message,logger) VALUES
('%d{yyyy-MM-dd HH:mm:ss}','%t','%.50p','%.50c','%.1000m','%.50l')" /> </layout> -->
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都 FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="LevelMin" value="DEBUG" />
<param name="LevelMax" value="FATAL" />
</filter>
</appender>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" />
<appender-ref ref="stdout" />
<!-- <appender-ref ref="file" /> -->
<!-- <appender-ref ref="file_log" /> -->
<!-- <appender-ref ref="MAIL" /> -->
<!-- <appender-ref ref="DATABASE" /> -->
</root>
</log4j:configuration>

32
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext-myBatis.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd">
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSource" />
</bean>
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="net.trustie.webmagic" />
</bean>
<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url" value="jdbc:mysql://192.168.80.104:3306/pages?characterEncoding=UTF-8" />
<property name="username" value="influx" />
<property name="password" value="influx1234" />
<property name="timeBetweenEvictionRunsMillis">
<value>21600000</value><!--6 hours-->
</property>
<property name="minEvictableIdleTimeMillis">
<value>21600000</value><!--connection的空闲时间大于这个值,就直接被关闭,并从连接池中删除-->
</property>
</bean>
</beans>

16
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/spring/applicationContext.xml
View File

@ -1,16 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
<context:annotation-config/>
<context:component-scan base-package="net.trustie.webmagic"/>
</beans>

43
crawler/dailyScheduledCrawler/fetch_networks/bin/resources/testDBlogger.xml
View File

@ -1,43 +0,0 @@
<!-- 输出通道"DATABASE"，输出方式是：输出所有级别的LOG到数据库 -->
<appender name="DATABASE" class="org.apache.log4j.jdbc.JDBCAppender">
<!--数据库的驱动，这里用的是MSSQL-->
<param name="driver" value="com.microsoft.jdbc.sqlserver.SQLServerDriver"/>
<!--这是Oracle用的驱动，在这里用MSSQL所以屏蔽掉了
<param name="driver" value="oracle.jdbc.driver.OracleDriver" />
-->
<!--要连接的数据库-->
<param name="URL" value="jdbc:microsoft:sqlserver://192.168.0.120:
1433;DatabaseName=test" />
<!--连接数据库的用户名-->
<param name="user" value="sa" />
<!--连接数据库的密码-->
<param name="password" value="sa" />
<!--向MSSQL数据库表LOG中插入数据的sql语句-->
<param name="sql" value=" INSERT INTO LOG
(LOGDATE,LOGLEVEL,LOGCLASS,LOGLOGGER,LOGMESSAGE)
values ('%d{yyyy-MM-dd HH:mm:ss}','%.50p', '%.50c',
'%.50l', '%.1000m')"/>
<!-- Oracle 的 insert 语句
<param name="sql" value=" insert into logrecord
(id,packageid,userid,syscodeid,info,logtime,loglevel)
values(?,?,?,?,?,to_date('%d{yyyy-MM-dd HH:mm:ss}',
'yyyy-MM-dd HH24:mi:ss'),'%p')" />
-->
<!-- 过滤输出时Log内容，在这里，LevelMin是ERROR，LevelMax都
FATAL，所以输出DEBUG级别到FATAL级别的LOG数据 -->
<filter class="org.apache.log4j.varia.LevelRangeFilter">
<param name="LevelMin" value="ERROR"/>
<param name="LevelMax" value="FATAL"/>
</filter>
</appender>

30
crawler/dailyScheduledCrawler/fetch_networks/bin/sample_start.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='freecode'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn48m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/slashdot.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='slashdot'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/softpedia.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='softpedia'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

29
crawler/dailyScheduledCrawler/fetch_networks/bin/sourceforge.sh
View File

@ -1,29 +0,0 @@
#!/bin/bash
SITE='sourceforge'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx128m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

30
crawler/dailyScheduledCrawler/fetch_networks/bin/stackoverflow.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
#!!!!!!!!!!!!与sites中对应文件文件名相同，不包括扩展名
SITE='stackoverflow'
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
find ./target/classes/spring |xargs rm -f -r
find ./target/classes/sites |xargs rm -f -r
tmp='./bin/resources'
tmp='./sites':$tmp
tmp='./target/classes':$tmp
tmp='./target/fetchnetworks-1.0-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn48m -Xmx512m -Xms128m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#-XX:MinPermSize=128m"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ListHtmlCrawler $SITE >>log/${SITE}_list.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.UrlExtractor $SITE >>log/${SITE}_url.log 2>&1 &
java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.DetailHtmlCrawler $SITE >>log/${SITE}_detail.log 2>&1 &
#java $JVM_ARGS -classpath $CLASSPATH net.trustie.webmagic.one.ErrorPageCrawler $SITE >>log/${SITE}_error_process.log 2>&1 &

15
crawler/dailyScheduledCrawler/fetch_networks/bin/start_all.sh
View File

@ -1,15 +0,0 @@
#!/bin/bash
sh bin/cnblogs_news.sh
sh bin/cnblogs_q_solved.sh
sh bin/cnblogs_q_unsolved.sh
sh bin/csdn_ask.sh
sh bin/csdn_blog.sh
sh bin/csdn_topic.sh
sh bin/dewen_question.sh
sh bin/iteye_ask.sh
sh bin/openhub.sh
sh bin/oschina_project.sh
sh bin/oschina_question.sh
sh bin/sourceforge.sh
sh bin/stackoverflow.sh

BIN
crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-core-fix.jar
View File

Binary file not shown.

BIN
crawler/dailyScheduledCrawler/fetch_networks/lib/webmagic-extension-fix.jar
View File

Binary file not shown.

25
crawler/dailyScheduledCrawler/fetch_networks/ossean-bak.xml
View File

@ -1,25 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<entry key="pageF">1</entry>
<entry key="domain">one</entry>
<entry key="threadNum">5</entry>
<entry key="startPageIndex">1</entry>
<entry key="isSpawnUrl">true</entry>
<entry key="endInPageIndex">20000</entry>
<entry key="retryTimes">5</entry>
<entry key="timeOut">30000</entry>
<entry key="sleepTimeThread">5000</entry>
<entry key="cycleRetryTimes">5</entry>
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<entry key="sleepTimeSpider">10000</entry>
<entry key="startUrl">http://www.oschina.net/question/</entry>
<entry key="urlList">http://www\.oschina\.net/question\?catalog=[12]\&show=active\&p=\d+</entry>
<entry key="urlPost">http://www\.oschina\.net/question/\d+_\d+</entry>
<entry key="offset">0</entry>
<entry key="limitLine">10</entry>
<entry key="noUrlWaitTime">10000</entry>
<entry key="postTableName">html_test</entry>
<entry key="listTableName">target_url</entry>
</properties>

202
crawler/dailyScheduledCrawler/fetch_networks/pom.xml
View File

@ -1,202 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>net.trustie</groupId>
<artifactId>fetchnetworks</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- m2eclipse wtp 0.12+ enabled to configure contextRoot, add by w.vela -->
<m2eclipse.wtp.contextRoot>/</m2eclipse.wtp.contextRoot>
<spring-version>3.1.1.RELEASE</spring-version>
<spring-security-version>3.1.0.RELEASE</spring-security-version>
</properties>
<build>
<sourceDirectory>${basedir}/src/main/java</sourceDirectory>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>xerces</groupId>
<artifactId>xerces</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20141113</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>14.0</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>xsoup</artifactId>
<version>0.2.4</version>
</dependency>
<dependency>
<groupId>com.github.dreamhead</groupId>
<artifactId>moco-core</artifactId>
<version>0.10.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.7</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>1.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>0.8.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.4</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>${spring-version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.18</version>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${spring-version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.trustie</groupId>
<artifactId>webmagic-core-fix</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/webmagic-core-fix.jar</systemPath>
</dependency>
<dependency>
<groupId>net.trustie</groupId>
<artifactId>webmagic-extension-fix</artifactId>
<version>1.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/webmagic-extension-fix.jar</systemPath>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>2.42.2</version>
</dependency>
</dependencies>
</project>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/51cto_blog.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>51cto</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>div.rightbox>div.r_li>h4>a</urlPost>
<fixAllRelativeHrefs>http://blog.51cto.com</fixAllRelativeHrefs>
<prefixUrl>http://blog.51cto.com/original/0/</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>51cto_blog</domain>
<minInterval>20000</minInterval>
<maxInterval>30000</maxInterval>
<minlongSleepTime>600000</minlongSleepTime>
<maxlongSleepTime>1200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

34
crawler/dailyScheduledCrawler/fetch_networks/sites/Copy of sample_site.xml
View File

@ -1,34 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">csdn_topic</entry>
<!-- 有序爬取中，起始页页码-->
<entry key="startPageIndex">1</entry>
<!-- 有序爬取中，结束页页码-->
<entry key="endInPageIndex">10</entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
<entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">50</entry>
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
<!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

30
crawler/dailyScheduledCrawler/fetch_networks/sites/apache.xml
View File

@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1</endInPageIndex>
<urlPost>table>tbody>tr>td.body>div>div>ul>li> a</urlPost>
<fixAllRelativeHrefs>http://projects.apache.org</fixAllRelativeHrefs>
<prefixUrl>http://projects.apache.org/indexes/alpha.html</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>864000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>apache</domain>
<minInterval>10000</minInterval>
<maxInterval>30000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of 51cto_blog.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">51cto_blog</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">30</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://blog.51cto.com/original/0/1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.rightbox>div.r_li>h4>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://blog.51cto.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://blog.51cto.com/original/0/</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">16</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of apache.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">apache</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">30</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://projects.apache.org/indexes/alpha.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table>tbody>tr>td.body>div>div>ul>li> a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://projects.apache.org/indexes/alpha.html</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://projects.apache.org/indexes/alpha.html</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">1</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">8640000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_news.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">cnblogs_news</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">4845</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://news.cnblogs.com/n/page/1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#news_list>div.news_block>div.content>h2.news_entry>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://news.cnblogs.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://news.cnblogs.com/n/page/</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">5</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_solved.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">cnblogs_q_solved</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">160</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">4000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://q.cnblogs.com/list/solved?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://q.cnblogs.com/list/solved?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

72
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of cnblogs_q_unsolved.xml
View File

@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">cnblogs_q_unsolved</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">160</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">4000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://q.cnblogs.com/list/unsolved?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.one_entity>div.news_item>h2.news_entry>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://q.cnblogs.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://q.cnblogs.com/list/unsolved?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of code_project.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">codeproject</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">21</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content-list-item>div.title>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.codeproject.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">1</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of dewen_question.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">dewen_question</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">1114</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.dewen.io/questions?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.dewen.io</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.dewen.io/questions?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">1</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">432000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of gna.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">gna</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">1348</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=1&amp;max_rows=1#results</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table>tbody>tr>td>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://gna.org</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">&amp;max_rows=1#results</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">200</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">432000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of lupaworld.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">lupaworld</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">2</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">479</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.lupaworld.com/forum-13746-2.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table#threadlisttableid>tbody>tr>th>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.lupaworld.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.lupaworld.com/forum-13746-</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.html</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">2592000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

71
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of openhub.xml
View File

@ -1,71 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- -->
<entry key="domain">openhub</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">66784</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">60000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">https://www.openhub.net/p?page=1&amp;q=&amp;sort=users</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.project>h2.title>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">60000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">https://www.openhub.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">https://www.openhub.net/p?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">&amp;q=&amp;sort=users</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">500</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">100000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of phpchina.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">phpchina</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">1289</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://bbs.phpchina.com/forum-17-1.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table#threadlisttableid>tbody>tr>th>a.s.xst</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://bbs.phpchina.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.phpchina.com/forum-17-</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.html</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">1296000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of softpedia.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">softpedia</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">30</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://linux.softpedia.com/index1.shtml</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#sjmp>div>h4>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://linux.softpedia.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://linux.softpedia.com/index</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.shtml</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">10</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of sourceforge.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- -->
<entry key="domain">sourceforge</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">17621</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">60000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">10000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://sourceforge.net/directory/?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">ul.projects>li>div.project_info>header>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://sourceforge.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://sourceforge.net/directory/?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">500</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">250000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/Copy of stackoverflow.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- 有序爬取时，每次生成页面链接数目-->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">stackoverflow</entry>
<!-- 线程数目-->
<entry key="threadNum">1</entry>
<!-- 有序爬取中，起始页页码-->
<entry key="startPageIndex">1</entry>
<!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
<entry key="isSpawnUrl">true</entry>
<!-- 有序爬取中，结束页页码-->
<entry key="endInPageIndex">25000</entry>
<!-- 页面下载超时，重试次数-->
<entry key="retryTimes">5</entry>
<!-- 下载时，连接超时等待时间-->
<entry key="timeOut">60000</entry>
<!-- 超时时间，单位是毫秒-->
<entry key="sleepTimeThread">0</entry>
<!-- 循环重试次数-->
<entry key="cycleRetryTimes">5</entry>
<!-- 请求头的UserAgent信息-->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
<entry key="sleepTimeSpider">10000</entry>
<!-- 起始种子URL -->
<entry key="startUrl">http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#mainbar>div#questions>div.question-summary>div.summary>h3>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
<entry key="noUrlWaitTime">10000</entry>
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
<entry key="fixAllRelativeHrefs">http://stackoverflow.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://stackoverflow.com/questions?pagesize=50&amp;sort=newest&amp;page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">250</entry>
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
<entry key="minInterval">2000</entry>
<entry key="maxInterval">3000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">2400000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">7200000</entry>
<entry key="MaxlongSleepInterval">14400000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/coypy of linuxtone.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">linuxtone</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">64</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://bbs.linuxtone.org/forum-15-1.html</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table>tbody>tr>th>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://bbs.linuxtone.org</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.linuxtone.org/forum-15-</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">.html</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">2592000</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/backup/jcopy of iteye_blog.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">iteye_blog</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">15</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.iteye.com/blogs?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content>h3>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.iteye.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.iteye.com/blogs?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">2</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/bytes.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>question</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2000</endInPageIndex>
<urlPost>table#threadslist>tbody>tr>td>div>div.threadbit_title>a</urlPost>
<fixAllRelativeHrefs>http://bytes.com</fixAllRelativeHrefs>
<prefixUrl>http://bytes.com/answers/</prefixUrl>
<postfixUrl>/</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>bytes</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

32
crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_news.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>4845</endInPageIndex>
<urlPost>div#news_list>div.news_block>div.content>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://news.cnblogs.com</fixAllRelativeHrefs>
<prefixUrl>http://news.cnblogs.com/n/page/</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>5</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>cnblogs_news</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

27
crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_solved.xml
View File

@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>160</endInPageIndex>
<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
<prefixUrl>http://q.cnblogs.com/list/solved?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>cnblogs_q_solved</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/cnblogs_q_unsolved.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>div.one_entity>div.news_item>h2.news_entry>a</urlPost>
<fixAllRelativeHrefs>http://q.cnblogs.com</fixAllRelativeHrefs>
<prefixUrl>http://q.cnblogs.com/list/unsolved?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>cnblogs_q_unsolved</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/code_project.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>259</endInPageIndex>
<urlPost>div.content-list-item>div.title>a</urlPost>
<fixAllRelativeHrefs>http://www.codeproject.com</fixAllRelativeHrefs>
<!--<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=30&amp;la_app=20&amp;la_minscore=3.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</prefixUrl>-->
<prefixUrl>http://www.codeproject.com/script/Articles/Latest.aspx?la_as=366&amp;la_app=20&amp;la_minscore=1.0&amp;la_allattr=False&amp;at=1%2c3%2c6%2c8&amp;pgnum=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>codeproject</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_ask.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2496</endInPageIndex>
<urlPost>div.common_con>div.questions_detail_con>dl>dt>a</urlPost>
<fixAllRelativeHrefs>http://ask.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://ask.csdn.net/?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>25</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>csdn_ask</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_blog.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>101</endInPageIndex>
<urlPost>div.main_center>div.blog_list>h1>a:not(.category)</urlPost>
<fixAllRelativeHrefs>http://blog.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://blog.csdn.net/index.html?&amp;page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>csdn_blog</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

81
crawler/dailyScheduledCrawler/fetch_networks/sites/csdn_topic.xml
View File

@ -1,81 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>mobile</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>808</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/Mobile?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>8</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>CloudComputing</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>22</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/CloudComputing?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>java</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2555</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/Java?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>4</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>web</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1853</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/WebDevelop?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>OtherLanguage</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>336</endInPageIndex>
<urlPost>div.wrap > div.content > table > tbody > tr> td.title > a</urlPost>
<fixAllRelativeHrefs>http://bbs.csdn.net</fixAllRelativeHrefs>
<prefixUrl>http://bbs.csdn.net/forums/OtherLanguage?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>csdn_topic</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

30
crawler/dailyScheduledCrawler/fetch_networks/sites/dewen_question.xml
View File

@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1114</endInPageIndex>
<urlPost>div.question_list>div.itemtop>ul.floatul>li.list_tt>span.question_listitle>a</urlPost>
<fixAllRelativeHrefs>http://www.dewen.io</fixAllRelativeHrefs>
<prefixUrl>http://www.dewen.io/questions?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>432000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>dewen_question</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

59
crawler/dailyScheduledCrawler/fetch_networks/sites/freecode.xml
View File

@ -1,59 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">5</entry>
<!-- -->
<entry key="domain">freecode</entry>
<!-- -->
<entry key="threadNum">5</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">4799</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">30000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">10000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.freecode.com/projects?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.project>h2.release-head>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">60000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.freecode.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.freecode.com/projects?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">5</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
</properties>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/gna.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1348</endInPageIndex>
<urlPost>table>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://gna.org</fixAllRelativeHrefs>
<prefixUrl>http://gna.org/search/?type_of_search=soft&amp;words=%2A&amp;type=1&amp;offset=</prefixUrl>
<postfixUrl>&amp;max_rows=1#results</postfixUrl>
<incrementalPages>100</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>gna</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_post.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>blog</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>138</endInPageIndex>
<urlPost>table>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
<prefixUrl>http://www.ibm.com/developerworks/cn/views/opensource/libraryview.jsp?site_id=10&amp;contentarea_by=%E6%89%80%E6%9C%89%E4%B8%93%E5%8C%BA&amp;sort_by=&amp;sort_order=2&amp;start=</prefixUrl>
<postfixUrl>00&amp;end=13874&amp;topic_by=-1&amp;product_by=&amp;type_by=%E6%89%80%E6%9C%89%E7%B1%BB%E5%88%AB&amp;show_abstract=false&amp;search_by=&amp;industry_by=&amp;series_title_by=</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>6048000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>ibm_post</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/ibm_project.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1</endInPageIndex>
<urlPost>table>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.ibm.com</fixAllRelativeHrefs>
<prefixUrl>http://www.ibm.com/developerworks/views/opensource/projects.jsp</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>864000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>ibm_project</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

59
crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_ask.xml
View File

@ -1,59 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">5</entry>
<!-- -->
<entry key="domain">iteye_ask</entry>
<!-- -->
<entry key="threadNum">5</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">5329</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">30000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://www.iteye.com/ask?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div#ask_list>div.question-summary>div.summary>h3>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">60000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://www.iteye.com/</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://www.iteye.com/ask?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">50</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
</properties>

78
crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_bbs.xml
View File

@ -1,78 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name>mobile</name>
<startPageIndex>z</startPageIndex>
<endInPageIndex>111</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/mobile?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>language</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>242</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/language?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>web</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>320</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/web?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>java</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>754</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/Java?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<name>tech</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>242</endInPageIndex>
<urlPost>table#forum_main>tbody>tr>td>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/forums/board/Tech?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>iteye_bbs</domain>
<minInterval>30000</minInterval>
<maxInterval>50000</maxInterval>
<minlongSleepTime>3600000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

28
crawler/dailyScheduledCrawler/fetch_networks/sites/iteye_blog.xml
View File

@ -1,28 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<name>blog</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>15</endInPageIndex>
<urlPost>div.content>h3>a</urlPost>
<fixAllRelativeHrefs>http://www.iteye.com</fixAllRelativeHrefs>
<prefixUrl>http://www.iteye.com/blogs?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>iteye_blog</domain>
<minInterval>30000</minInterval>
<maxInterval>50000</maxInterval>
<minlongSleepTime>3600000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/javaforge.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">1</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">javaforge</entry>
<!-- -->
<entry key="threadNum">1</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">31</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">5000</entry>
<!-- 起始页 -->
<entry key="startUrl">http://javaforge.com/listProjects.spr?page=1&amp;displayAllProjects=on</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">table#entry>tbody>tr>td:nth-child(1) > a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://javaforge.com</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://javaforge.com/listProjects.spr?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl">&amp;displayAllProjects=on</entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">10</entry>
<!-- 列表页更新完后休息时间 单位为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">86400</entry>
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/lagou.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>houduan</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost> #container > div.content>ul> li> div> div > a</urlPost>
<fixAllRelativeHrefs>http://www.lagou.com</fixAllRelativeHrefs>
<prefixUrl>http://www.lagou.com/jobs/list_%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91?kd=%E5%90%8E%E7%AB%AF%E5%BC%80%E5%8F%91&amp;spc=1&amp;pl=&amp;gj=&amp;xl=&amp;yx=&amp;gx=&amp;st=&amp;labelWords=label&amp;lc=&amp;workAddress=&amp;city=%E5%85%A8%E5%9B%BD&amp;requestId=&amp;pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>lagou</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

81
crawler/dailyScheduledCrawler/fetch_networks/sites/linuxtone.xml
View File

@ -1,81 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>System</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>64</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-15-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>server</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-22-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>dbserver</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>32</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-24-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>yngz</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>32</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-49-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>fzjh</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>24</endInPageIndex>
<urlPost>table>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://bbs.linuxtone.org</fixAllRelativeHrefs>
<prefixUrl>http://bbs.linuxtone.org/forum-26-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>linuxtone</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

31
crawler/dailyScheduledCrawler/fetch_networks/sites/lupaworld.xml
View File

@ -1,31 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>479</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a</urlPost>
<fixAllRelativeHrefs>http://www.lupaworld.com</fixAllRelativeHrefs>
<prefixUrl>http://www.lupaworld.com/forum-13746-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>2592000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>lupaworld</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/neitui.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>610</endInPageIndex>
<urlPost> ul>li>div.cont>div.jobnote.clearfix > div.jobnote-l>a</urlPost>
<fixAllRelativeHrefs>http://www.neitui.me</fixAllRelativeHrefs>
<prefixUrl>http://www.neitui.me/neitui/type=all&amp;page=</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>16</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>neitui</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

152
crawler/dailyScheduledCrawler/fetch_networks/sites/open_open.xml
View File

@ -1,152 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>dev</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>370</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/1?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>language</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>652</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/36?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>website</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>41</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/57?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>server</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>145</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/75?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>database</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>102</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/90?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>plugin</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>53</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/52?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>OS</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>42</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/155?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>manage</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>12</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/319?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>dev_manage</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>9</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/159?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>util</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>7</endInPageIndex>
<urlPost>div#list-files>table>tbody>tr>td>table>tbody>tr>td>h2>a</urlPost>
<fixAllRelativeHrefs>http://www.open-open.com</fixAllRelativeHrefs>
<prefixUrl>http://www.open-open.com/lib/list/322?pn=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>open_open</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

35
crawler/dailyScheduledCrawler/fetch_networks/sites/openhub.xml
View File

@ -1,35 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>66784</endInPageIndex>
<urlPost>div.well>h2.title>a</urlPost>
<fixAllRelativeHrefs>https://www.openhub.net</fixAllRelativeHrefs>
<!--<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
<postfixUrl>&amp;q=&amp;sort=users</postfixUrl>
<incrementalPages>300</incrementalPages>
<incrementSleepTime>300000</incrementSleepTime> -->
<prefixUrl>https://www.openhub.net/p?page=</prefixUrl>
<postfixUrl>&amp;q=&amp;sort=new</postfixUrl>
<incrementalPages>8</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>openhub</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

29
crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_project.xml
View File

@ -1,29 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2044</endInPageIndex>
<urlPost>div.ProjectList>ul.List>li>h3>a</urlPost>
<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
<prefixUrl>http://www.oschina.net/project/list?prefix=&amp;sort=time&amp;p=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>oschina_project</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

31
crawler/dailyScheduledCrawler/fetch_networks/sites/oschina_question.xml
View File

@ -1,31 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>2361</endInPageIndex>
<urlPost>ul.list>li.question>div.question-detail>strong>a</urlPost>
<fixAllRelativeHrefs>http://www.oschina.net</fixAllRelativeHrefs>
<prefixUrl>http://www.oschina.net/question?catalog=1&amp;show=active&amp;p=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>oschina_question</domain>
<minInterval>30000</minInterval>
<maxInterval>60000</maxInterval>
<minlongSleepTime>7200000</minlongSleepTime>
<maxlongSleepTime>10800000</maxlongSleepTime>
<minlongSleepInterval>1800000</minlongSleepInterval>
<maxlongSleepInterval>3600000</maxlongSleepInterval>
</properties>
</site>

59
crawler/dailyScheduledCrawler/fetch_networks/sites/ossean.xml
View File

@ -1,59 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- -->
<entry key="pageF">3</entry>
<!-- -->
<entry key="domain">ossean</entry>
<!-- -->
<entry key="threadNum">3</entry>
<!-- -->
<entry key="startPageIndex">1</entry>
<!-- -->
<entry key="isSpawnUrl">true</entry>
<!-- -->
<entry key="endInPageIndex">5777</entry>
<!-- -->
<entry key="retryTimes">5</entry>
<!-- -->
<entry key="timeOut">10000</entry>
<!-- -->
<entry key="sleepTimeThread">0</entry>
<!-- -->
<entry key="cycleRetryTimes">5</entry>
<!-- -->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- -->
<entry key="sleepTimeSpider">500</entry>
<!-- 起始页 -->
<entry key="startUrl">http://localhost/open_source_projects?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">ul.projects>li.project-table>div.root>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- -->
<entry key="noUrlWaitTime">10000</entry>
<!-- -->
<entry key="fixAllRelativeHrefs">http://localhost</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://localhost/open_source_projects?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">5</entry>
<!-- 列表页更新完后休息时间 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">0</entry>
</properties>

70
crawler/dailyScheduledCrawler/fetch_networks/sites/phpchina.xml
View File

@ -1,70 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>ask</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>1289</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-17-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>server_os</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>91</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-195-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>web</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>81</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-213-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>db</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>43</endInPageIndex>
<urlPost>table#threadlisttableid>tbody>tr>th>a.s.xst</urlPost>
<fixAllRelativeHrefs>http://bbs.phpchina.com</fixAllRelativeHrefs>
<prefixUrl>http://bbs.phpchina.com/forum-196-</prefixUrl>
<postfixUrl>.html</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>1296000</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>phpchina</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

72
crawler/dailyScheduledCrawler/fetch_networks/sites/sample_site.xml
View File

@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>Spider</comment>
<!-- 有序爬取时，每次生成页面链接数目-->
<entry key="pageF">3</entry>
<!-- 爬虫所爬内容 最好定义与脚本中的SITE名字相同的名字 -->
<entry key="domain">csdn_topic</entry>
<!-- 线程数目-->
<entry key="threadNum">3</entry>
<!-- 有序爬取中，起始页页码-->
<entry key="startPageIndex">1</entry>
<!-- 是否启用自动发现链接（false状态下只爬取给定链接）注解模式下有效-->
<entry key="isSpawnUrl">true</entry>
<!-- 有序爬取中，结束页页码-->
<entry key="endInPageIndex">10</entry>
<!-- 页面下载超时，重试次数-->
<entry key="retryTimes">5</entry>
<!-- 下载时，连接超时等待时间-->
<entry key="timeOut">10000</entry>
<!-- 单位是毫秒-->
<entry key="sleepTimeThread">0</entry>
<!-- 循环重试次数-->
<entry key="cycleRetryTimes">5</entry>
<!-- 请求头的UserAgent信息-->
<entry key="userAgent">Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)</entry>
<!-- 增量更新时，每循环一次（设定的起始页到结束页），休眠时间，单位毫秒-->
<entry key="sleepTimeSpider">4000</entry>
<!-- 起始种子URL -->
<entry key="startUrl">http://bbs.csdn.net/tech_hot_topics?page=1</entry>
<!-- 列表页的规则 -->
<entry key="urlList"></entry>
<!-- 详情页链接抽取规则 -->
<entry key="urlPost">div.content>div.list_1>ul>li>a</entry>
<!-- -->
<entry key="offset">0</entry>
<!-- -->
<entry key="limitLine">10</entry>
<!-- 数据库队列无连接时等待抽取的时间，单位毫秒-->
<entry key="noUrlWaitTime">10000</entry>
<!-- 用于将页面原始RawText中的相对路径修复成绝对路径-->
<entry key="fixAllRelativeHrefs">http://bbs.csdn.net</entry>
<!-- 列表页url规则的前缀 -->
<entry key="prefixUrl">http://bbs.csdn.net/tech_hot_topics?page=</entry>
<!-- 列表页url规则的后缀 -->
<entry key="postfixUrl"></entry>
<!-- 详情页抽取方式，对应的抽取规则存在urlPost字段中，可选项有 XPath定位到a标签的href属性;CSSPath按照jsoup方式抽取，定位到a标签;URLPattern详情页url的规则 -->
<entry key="extractMethod">CSSPath</entry>
<!-- 代理ip -->
<entry key="proxyIp">192.168.245.1</entry>
<!-- 代理端口号 -->
<entry key="proxyPort">3128</entry>
<!-- 爬取模式 0:mirror 或者是 1:increment-->
<entry key="mode">0</entry>
<!-- 增量模式观察的页数 -->
<entry key="incrementalPages">50</entry>
<!-- 列表页更新完后休息时间 单位 为秒 为了给足够时间下载更新的详情页 -->
<entry key="incrementSleepTime">20000</entry>
<!-- 每次请求间隔的 最大值和最小值。请求时从这区间随机选取一个 -->
<entry key="minInterval">10000</entry>
<entry key="maxInterval">20000</entry>
<!--长时间睡眠时候的睡眠时间的上下限 -->
<entry key="minlongSleepTime">1800000</entry>
<entry key="MaxlongSleepTime">7200000</entry>
<!--长时间睡眠时候的睡眠间隔的上下限 -->
<entry key="minlongSleepInterval">3600000</entry>
<entry key="MaxlongSleepInterval">7200000</entry>
</properties>

32
crawler/dailyScheduledCrawler/fetch_networks/sites/slashdot.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name></name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>66</endInPageIndex>
<urlPost> #firehoselist>article>header>h2>span>a</urlPost>
<fixAllRelativeHrefs>http://slashdot.org</fixAllRelativeHrefs>
<prefixUrl>http://slashdot.org/?page=</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>slashdot</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

134
crawler/dailyScheduledCrawler/fetch_networks/sites/softpedia.xml
View File

@ -1,134 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<site>
<subSites>
<subSite>
<!-- 普通网站name不用写 -->
<name>linux</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://linux.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://linux.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>win</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://win.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://win.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>mac</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>35</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mac.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mac.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>5</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>windows-phone</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>46</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mobile.softpedia.com/windows-phone,</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>5</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>android</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>35</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mobile.softpedia.com/android,</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>ios</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>22</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://mobile.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://mobile.softpedia.com/ios,</prefixUrl>
<postfixUrl></postfixUrl>
<incrementalPages>1</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>driver</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://drivers.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://drivers.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>10</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>game</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://games.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://games.softpedia.com/index</prefixUrl>
<postfixUrl>.shtml</postfixUrl>
<incrementalPages>2</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
<subSite>
<!-- 普通网站name不用写 -->
<name>webscripts</name>
<startPageIndex>1</startPageIndex>
<endInPageIndex>30</endInPageIndex>
<urlPost>h4>a</urlPost>
<fixAllRelativeHrefs>http://webscripts.softpedia.com</fixAllRelativeHrefs>
<prefixUrl>http://webscripts.softpedia.com/index/latest-scripts-3</prefixUrl>
<postfixUrl>-0-0.html</postfixUrl>
<incrementalPages>3</incrementalPages>
<incrementSleepTime>86400</incrementSleepTime>
<mode>0</mode>
</subSite>
</subSites>
<properties>
<domain>softpedia</domain>
<minInterval>10000</minInterval>
<maxInterval>20000</maxInterval>
<minlongSleepTime>1800000</minlongSleepTime>
<maxlongSleepTime>7200000</maxlongSleepTime>
<minlongSleepInterval>3600000</minlongSleepInterval>
<maxlongSleepInterval>7200000</maxlongSleepInterval>
</properties>
</site>

Some files were not shown because too many files have changed in this diff Show More