kevinli
/
educoder
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix the extractor of the openhub_homepage(by liuyizhou)

This commit is contained in:
wrzzx 2017-02-15 21:51:07 +08:00
parent ae32cea6d8
commit 91137d2ac4
23 changed files with 1034 additions and 837 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
new_osseanextractor/.idea/encodings.xml
View File

@ -1,7 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" native2AsciiForPropertiesFiles="true" defaultCharsetForPropertiesFiles="UTF-8">
<component name="Encoding">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
<file url="PROJECT" charset="UTF-8" />
</component>
</project>

3
new_osseanextractor/.idea/misc.xml
View File

@ -1,5 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="EntryPointsManager">
<entry_points version="2.0" />
</component>
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>

2
new_osseanextractor/.idea/vcs.xml
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
<mapping directory="" vcs="Git" />
</component>
</project>

740
new_osseanextractor/.idea/workspace.xml
View File

@ -1,10 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="08302292-1670-461c-87ed-da9c7726ec87" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/osseanextractor.iml" afterPath="$PROJECT_DIR$/osseanextractor.iml" />
</list>
<list default="true" id="08302292-1670-461c-87ed-da9c7726ec87" name="Default" comment="" />
<ignored path="osseanextractor.iws" />
<ignored path=".idea/workspace.xml" />
<ignored path="$PROJECT_DIR$/target/" />
@ -28,39 +25,58 @@
<file leaf-file-name="Application.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/Application.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="463">
<caret line="252" column="5" lean-forward="true" selection-start-line="252" selection-start-column="5" selection-end-line="252" selection-end-column="5" />
<state relative-caret-position="17">
<caret line="71" column="31" lean-forward="false" selection-start-line="71" selection-start-column="31" selection-end-line="71" selection-end-column="31" />
<folding>
<element signature="e#1153#1169#0" expanded="true" />
<element signature="e#1381#1397#0" expanded="true" />
<element signature="e#2807#2815#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="openhub_Model.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/model/openhub_Model.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6681">
<caret line="422" column="70" lean-forward="false" selection-start-line="422" selection-start-column="70" selection-end-line="422" selection-end-column="70" />
<folding>
<element signature="e#7633#7634#0" expanded="true" />
<element signature="e#7656#7657#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="OpenhubReExtractor.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/OpenhubReExtractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="646">
<caret line="57" column="60" lean-forward="false" selection-start-line="57" selection-start-column="60" selection-end-line="57" selection-end-column="60" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="ExtractMutilLink4Openhub.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1530">
<caret line="97" column="35" lean-forward="false" selection-start-line="97" selection-start-column="35" selection-end-line="97" selection-end-column="35" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>starlee</find>
<find>saveresult</find>
<find>afterpro</find>
<find>getHo</find>
<find>getho</find>
<find>excute</find>
<find>gethome</find>
<find>gethomepage</find>
<find>sethomepage</find>
<find>seth</find>
<find>this.hom</find>
<find>afterp</find>
<find>homepage</find>
<find>cleanho</find>
<find>afterproce</find>
<find>afterpr</find>
<find>deal</find>
</findStrings>
<dirStrings>
<dir>D:\ossean\new_osseanextractor</dir>
</dirStrings>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/.." />
@ -74,11 +90,8 @@
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java" />
<option value="$PROJECT_DIR$/src/main/java/net/trustie/one/Application.java" />
<option value="$PROJECT_DIR$/src/main/java/core/PageModelExtractor.java" />
<option value="$PROJECT_DIR$/src/main/java/net/trustie/model/openhub_Model.java" />
<option value="$PROJECT_DIR$/resources/spring/applicationContext-myBatis.xml" />
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/java/net/trustie/one/Application.java" />
</list>
</option>
</component>
@ -102,59 +115,13 @@
</MavenImportingSettings>
</option>
</component>
<component name="MavenProjectNavigator">
<treeState>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$RootNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$ProjectNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$RootNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$ProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="Plugins" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$PluginsNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$RootNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$ProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="Plugins" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$PluginsNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="clean" />
<option name="myItemType" value="org.jetbrains.idea.maven.navigator.MavenProjectsStructure$PluginNode" />
</PATH_ELEMENT>
</PATH>
</treeState>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="-8" />
<option name="y" value="-8" />
<option name="width" value="1382" />
<option name="height" value="744" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true">
<component name="ProjectLevelVcsManager">
<ConfirmationsSetting value="1" id="Add" />
</component>
<component name="ProjectView">
@ -174,7 +141,6 @@
<panes>
<pane id="Scratches" />
<pane id="PackagesPane" />
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<PATH>
@ -201,6 +167,84 @@
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="new_osseanextractor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="trustie" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="new_osseanextractor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="trustie" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="one" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="new_osseanextractor" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="osseanextractor" />
@ -221,6 +265,7 @@
</PATH>
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
@ -242,16 +287,14 @@
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeOrder2" value="2" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeWidth3" value="927" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeOrder3" value="3" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../../new_osseanextractor" />
<property name="nodejs_interpreter_path" value="C:/Program Files (x86)/nodejs/node" />
<property name="js.eslint.eslintPackage" value="" />
<property name="js-jscs-nodeInterpreter" value="C:\Program Files (x86)\nodejs\node.exe" />
<property name="settings.editor.selected.configurable" value="MavenSettings" />
<property name="SearchEverywhereHistoryKey" value="&#9;FILE&#9;file://D:/ossean/new_osseanextractor/src/main/java/net/trustie/one/Application.java&#10;homepage&#9;PSI&#9;JAVA://net.trustie.model.freecode_Model#homepage" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="C:\Users\zaihuilvcha\Desktop\ossean\new_osseanextractor\target\classes" />
<recent name="C:\Users\zaihuilvcha\Desktop\ossean\new_osseanextractor\resources\spring" />
</key>
</component>
<component name="RunManager" selected="Application.Application">
@ -275,6 +318,26 @@
<envs />
<method />
</configuration>
<configuration default="false" name="OpenhubReExtractor" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea">
<pattern>
<option name="PATTERN" value="net.trustie.one.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<option name="MAIN_CLASS_NAME" value="net.trustie.one.OpenhubReExtractor" />
<option name="VM_PARAMETERS" />
<option name="PROGRAM_PARAMETERS" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="ENABLE_SWING_INSPECTOR" value="false" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<module name="osseanextractor" />
<envs />
<method />
</configuration>
<configuration default="true" type="AndroidRunConfigurationType" factoryName="Android App">
<module name="" />
<option name="DEPLOY" value="true" />
@ -584,28 +647,6 @@
<envs />
<method />
</configuration>
<configuration default="true" type="MavenRunConfiguration" factoryName="Maven">
<MavenSettings>
<option name="myGeneralSettings" />
<option name="myRunnerSettings" />
<option name="myRunnerParameters">
<MavenRunnerParameters>
<option name="profiles">
<set />
</option>
<option name="goals">
<list />
</option>
<option name="profilesMap">
<map />
</option>
<option name="resolveToWorkspace" value="false" />
<option name="workingDirPath" value="" />
</MavenRunnerParameters>
</option>
</MavenSettings>
<method />
</configuration>
<configuration default="true" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
@ -663,12 +704,14 @@
<configuration default="true" type="osgi.bnd.run" factoryName="Test Launcher (JUnit)">
<method />
</configuration>
<list size="1">
<list size="2">
<item index="0" class="java.lang.String" itemvalue="Application.Application" />
<item index="1" class="java.lang.String" itemvalue="Application.OpenhubReExtractor" />
</list>
<recent_temporary>
<list size="1">
<list size="2">
<item index="0" class="java.lang.String" itemvalue="Application.Application" />
<item index="1" class="java.lang.String" itemvalue="Application.OpenhubReExtractor" />
</list>
</recent_temporary>
<configuration default="true" type="AndroidRunConfigurationType" factoryName="Android Application">
@ -707,52 +750,49 @@
<updated>1432444136539</updated>
<workItem from="1432444138435" duration="203000" />
<workItem from="1436271132941" duration="1652000" />
<workItem from="1486629057644" duration="4995000" />
<workItem from="1486695212543" duration="8142000" />
<workItem from="1486710751207" duration="57266000" />
<workItem from="1487128073544" duration="1668000" />
<workItem from="1487131115495" duration="3684000" />
<workItem from="1487134909625" duration="10159000" />
<workItem from="1487166420461" duration="101000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="77610000" />
<option name="totallyTimeSpent" value="12115000" />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1382" height="744" extended-state="7" />
<frame x="-8" y="-8" width="1382" height="744" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Nl-Palette" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32933104" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32933104" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32590052" sideWeight="0.5030257" order="8" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32980332" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.7684391" sideWeight="0.5" order="12" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32590052" sideWeight="0.4969743" order="9" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.3270548" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.6226415" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Properties" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Spring" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32933104" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" />
<window_info id="Spring" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32933104" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.1913767" sideWeight="0.4819897" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25037822" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.24962178" sideWeight="0.4893048" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Theme Preview" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.35506004" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.18532526" sideWeight="0.5180103" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.5728988" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="SLIDING" type="SLIDING" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Application Servers" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="11" side_tool="false" content_ui="tabs" />
<window_info id="Application Servers" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32933104" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
@ -771,378 +811,58 @@
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" type="java-line">
<url>file://$PROJECT_DIR$/src/main/java/net/trustie/one/Application.java</url>
<line>154</line>
<properties />
<option name="timeStamp" value="37" />
</line-breakpoint>
</breakpoints>
<option name="time" value="46" />
<option name="time" value="3" />
</breakpoint-manager>
<watches-manager>
<configuration name="Application">
<watch expression="pages" />
<watch expression="getHomepage()" />
</configuration>
</watches-manager>
<watches-manager />
</component>
<component name="antWorkspaceConfiguration">
<option name="IS_AUTOSCROLL_TO_SOURCE" value="false" />
<option name="FILTER_TARGETS" value="false" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/resources/spring/applicationContext-myBatis.xml">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/Application.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="102">
<caret line="6" column="4" lean-forward="false" selection-start-line="6" selection-start-column="4" selection-end-line="6" selection-end-column="4" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/commons-dbcp/commons-dbcp/1.4/commons-dbcp-1.4-sources.jar!/org/apache/commons/dbcp/DriverConnectionFactory.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="391">
<caret line="41" column="40" lean-forward="true" selection-start-line="41" selection-start-column="40" selection-end-line="41" selection-end-column="40" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java" />
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java" />
<entry file="file://$PROJECT_DIR$/src/main/java/core/ExtractorBegin.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="34">
<caret line="2" column="17" lean-forward="false" selection-start-line="2" selection-start-column="17" selection-end-line="2" selection-end-column="17" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/dao/IteyeAsk_Dao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="68">
<caret line="4" column="0" lean-forward="false" selection-start-line="4" selection-start-column="0" selection-end-line="4" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://C:/Program Files/Java/jdk1.8.0_65/src.zip!/java/lang/Thread.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="125">
<caret line="140" column="6" lean-forward="false" selection-start-line="140" selection-start-column="6" selection-end-line="140" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/bin/resources/spring/applicationContext-myBatis.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="187">
<caret line="11" column="59" lean-forward="true" selection-start-line="11" selection-start-column="59" selection-end-line="11" selection-end-column="59" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/commons-dbcp/commons-dbcp/1.4/commons-dbcp-1.4-sources.jar!/org/apache/commons/dbcp/DriverConnectionFactory.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="361">
<caret line="45" column="75" lean-forward="false" selection-start-line="45" selection-start-column="75" selection-end-line="45" selection-end-column="75" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/mysql/mysql-connector-java/5.1.18/mysql-connector-java-5.1.18.jar!/com/mysql/jdbc/Util.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-8">
<caret line="227" column="0" lean-forward="false" selection-start-line="227" selection-start-column="0" selection-end-line="227" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/test/java/net/trustie/core/OsseanExtractorTest.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="68">
<caret line="6" column="13" lean-forward="false" selection-start-line="6" selection-start-column="13" selection-end-line="6" selection-end-column="13" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/model/apache_Model.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="18" column="5" lean-forward="false" selection-start-line="18" selection-start-column="5" selection-end-line="18" selection-end-column="5" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/model/freecode_Model.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="166">
<caret line="67" column="27" lean-forward="false" selection-start-line="67" selection-start-column="19" selection-end-line="67" selection-end-column="27" />
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#4415#4416#0" expanded="false" />
<element signature="e#4434#4435#0" expanded="false" />
<element signature="e#1381#1397#0" expanded="true" />
<element signature="e#2807#2815#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/downloader/ErrorPageDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="68">
<caret line="5" column="17" lean-forward="false" selection-start-line="5" selection-start-column="17" selection-end-line="5" selection-end-column="17" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/downloader/DataBasePageErrorOutPut.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-102">
<caret line="13" column="13" lean-forward="false" selection-start-line="13" selection-start-column="13" selection-end-line="13" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/downloader/GenerateRawPage.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="85">
<caret line="10" column="13" lean-forward="false" selection-start-line="10" selection-start-column="13" selection-end-line="10" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/downloader/PageDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="85">
<caret line="10" column="26" lean-forward="true" selection-start-line="10" selection-start-column="26" selection-end-line="10" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/dao/OpenHubProject_Dao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="392">
<caret line="39" column="25" lean-forward="false" selection-start-line="39" selection-start-column="25" selection-end-line="39" selection-end-column="25" />
</state>
</provider>
</entry>
<entry file="jar://C:/Program Files/Java/jdk1.8.0_65/src.zip!/java/util/concurrent/Executor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="401">
<caret line="138" column="7" lean-forward="true" selection-start-line="138" selection-start-column="7" selection-end-line="138" selection-end-column="7" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/mybatis/mybatis/3.1.1/mybatis-3.1.1-sources.jar!/org/apache/ibatis/binding/MapperProxy.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="191">
<caret line="43" column="0" lean-forward="false" selection-start-line="43" selection-start-column="0" selection-end-line="43" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/pipeline/OpenHubProject_pipeline.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="204">
<caret line="19" column="45" lean-forward="true" selection-start-line="19" selection-start-column="45" selection-end-line="19" selection-end-column="45" />
</state>
</provider>
</entry>
<entry file="jar://C:/Program Files/Java/jdk1.8.0_65/src.zip!/java/lang/Class.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="422">
<caret line="1386" column="16" lean-forward="true" selection-start-line="1386" selection-start-column="16" selection-end-line="1386" selection-end-column="16" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/us/codecraft/webmagic-core/0.5.2/webmagic-core-0.5.2-sources.jar!/us/codecraft/webmagic/selector/BaseElementSelector.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="132">
<caret line="25" column="0" lean-forward="false" selection-start-line="25" selection-start-column="0" selection-end-line="25" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.7.2/jsoup-1.7.2-sources.jar!/org/jsoup/Jsoup.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="115">
<caret line="57" column="0" lean-forward="false" selection-start-line="57" selection-start-column="0" selection-end-line="57" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.7.2/jsoup-1.7.2-sources.jar!/org/jsoup/parser/TreeBuilder.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="119">
<caret line="13" column="0" lean-forward="false" selection-start-line="13" selection-start-column="0" selection-end-line="13" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.7.2/jsoup-1.7.2-sources.jar!/org/jsoup/helper/DescendableLinkedList.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="238">
<caret line="16" column="0" lean-forward="false" selection-start-line="16" selection-start-column="0" selection-end-line="16" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.7.2/jsoup-1.7.2-sources.jar!/org/jsoup/parser/HtmlTreeBuilder.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="107">
<caret line="31" column="0" lean-forward="false" selection-start-line="31" selection-start-column="0" selection-end-line="31" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.7.2/jsoup-1.7.2-sources.jar!/org/jsoup/parser/ParseErrorList.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="239">
<caret line="15" column="0" lean-forward="false" selection-start-line="15" selection-start-column="0" selection-end-line="15" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/core/FieldExtractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="107">
<caret line="33" column="0" lean-forward="false" selection-start-line="33" selection-start-column="0" selection-end-line="33" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/core/Extractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="341">
<caret line="45" column="0" lean-forward="false" selection-start-line="45" selection-start-column="0" selection-end-line="45" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.7.2/jsoup-1.7.2-sources.jar!/org/jsoup/parser/Parser.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1152">
<caret line="89" column="0" lean-forward="false" selection-start-line="89" selection-start-column="0" selection-end-line="89" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/us/codecraft/webmagic-extension/0.5.2/webmagic-extension-0.5.2.jar!/us/codecraft/webmagic/model/annotation/ExtractBy.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="493">
<caret line="35" column="14" lean-forward="true" selection-start-line="35" selection-start-column="14" selection-end-line="35" selection-end-column="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/core/Page.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="799">
<caret line="54" column="16" lean-forward="false" selection-start-line="54" selection-start-column="16" selection-end-line="54" selection-end-column="16" />
<folding>
<element signature="e#709#710#0" expanded="false" />
<element signature="e#743#744#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/Extractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="595">
<caret line="43" column="0" lean-forward="false" selection-start-line="43" selection-start-column="0" selection-end-line="43" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/core/ModelPageProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="109">
<caret line="25" column="0" lean-forward="false" selection-start-line="25" selection-start-column="0" selection-end-line="25" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-lang3/3.1/commons-lang3-3.1-sources.jar!/org/apache/commons/lang3/StringUtils.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="130">
<caret line="221" column="26" lean-forward="false" selection-start-line="221" selection-start-column="26" selection-end-line="221" selection-end-column="26" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/utils/StringHandler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="172">
<caret line="72" column="26" lean-forward="false" selection-start-line="72" selection-start-column="26" selection-end-line="72" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/core/PageModelExtractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="738">
<caret line="322" column="0" lean-forward="false" selection-start-line="322" selection-start-column="0" selection-end-line="322" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/core/ResultItems.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="289">
<caret line="19" column="33" lean-forward="false" selection-start-line="19" selection-start-column="33" selection-end-line="19" selection-end-column="33" />
<folding>
<element signature="e#1303#1304#0" expanded="false" />
<element signature="e#1321#1322#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/extension/RawPage.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1428">
<caret line="103" column="0" lean-forward="false" selection-start-line="103" selection-start-column="0" selection-end-line="103" selection-end-column="0" />
<folding>
<element signature="e#1996#1997#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/extension/PageDownloader.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="136">
<caret line="8" column="26" lean-forward="false" selection-start-line="8" selection-start-column="26" selection-end-line="8" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/springframework/spring-beans/3.1.1.RELEASE/spring-beans-3.1.1.RELEASE-sources.jar!/org/springframework/beans/factory/support/AbstractBeanDefinitionReader.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2674">
<caret line="207" column="0" lean-forward="false" selection-start-line="207" selection-start-column="0" selection-end-line="207" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/resources/spring/applicationContext-myBatis.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="520">
<caret line="37" column="11" lean-forward="false" selection-start-line="37" selection-start-column="11" selection-end-line="37" selection-end-column="11" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/springframework/spring-core/3.1.1.RELEASE/spring-core-3.1.1.RELEASE.jar!/org/springframework/core/io/ClassPathResource.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="466">
<caret line="115" column="34" lean-forward="false" selection-start-line="115" selection-start-column="34" selection-end-line="115" selection-end-column="34" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/model/openhub_Model.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="495">
<caret line="868" column="5" lean-forward="false" selection-start-line="868" selection-start-column="5" selection-end-line="868" selection-end-column="5" />
<state relative-caret-position="7174">
<caret line="422" column="70" lean-forward="true" selection-start-line="422" selection-start-column="70" selection-end-line="422" selection-end-column="70" />
<folding>
<element signature="imports" expanded="true" />
<element signature="method#getAElements#0;class#openhub_Model#0" expanded="false" />
<element signature="method#handleNutShell0#0;class#openhub_Model#0" expanded="false" />
<element signature="method#handleNutShell1#0;class#openhub_Model#0" expanded="false" />
<element signature="method#handleNutShell2#0;class#openhub_Model#0" expanded="false" />
<element signature="method#handleNutShell3#0;class#openhub_Model#0" expanded="false" />
<element signature="method#handleDateAt#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setMostWrittenHeader#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setCommentsPercentageTail#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setCodebaseStatusHeaderTail#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setEstimateEffortTimeTail#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setFirstCommitTimeHeader#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setLastCommitTimeHeader#0;class#openhub_Model#0" expanded="false" />
<element signature="method#getHomepage#0;class#openhub_Model#0" expanded="false" />
<element signature="method#matchLink#0;class#openhub_Model#0" expanded="false" />
<element signature="method#setHomepage#0;class#openhub_Model#0" expanded="false" />
<element signature="e#7633#7634#0" expanded="true" />
<element signature="e#7656#7657#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tasks.txt">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/OpenhubReExtractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="969">
<caret line="57" column="60" lean-forward="true" selection-start-line="57" selection-start-column="60" selection-end-line="57" selection-end-column="60" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1530">
<caret line="97" column="35" lean-forward="true" selection-start-line="97" selection-start-column="35" selection-end-line="97" selection-end-column="35" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java" />
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java" />
<entry file="file://$PROJECT_DIR$/resources/spring/applicationContext-myBatis.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
@ -1150,44 +870,64 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/target/classes/spring/applicationContext.xml">
<entry file="file://$PROJECT_DIR$/bin/start.sh">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="221">
<caret line="13" column="0" lean-forward="false" selection-start-line="13" selection-start-column="0" selection-end-line="13" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.classpath">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="459">
<caret line="27" column="0" lean-forward="false" selection-start-line="27" selection-start-column="0" selection-end-line="27" selection-end-column="0" />
<folding />
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/resources/spring/applicationContext.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="204">
<caret line="12" column="56" lean-forward="false" selection-start-line="12" selection-start-column="56" selection-end-line="12" selection-end-column="56" />
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<entry file="jar://C:/Program Files/Java/jdk1.8.0_65/src.zip!/java/util/concurrent/Executor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="34">
<caret line="23" column="40" lean-forward="true" selection-start-line="23" selection-start-column="40" selection-end-line="23" selection-end-column="40" />
<state relative-caret-position="571">
<caret line="139" column="9" lean-forward="false" selection-start-line="139" selection-start-column="9" selection-end-line="139" selection-end-column="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1530">
<caret line="97" column="35" lean-forward="false" selection-start-line="97" selection-start-column="35" selection-end-line="97" selection-end-column="35" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/OpenhubReExtractor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="646">
<caret line="57" column="60" lean-forward="false" selection-start-line="57" selection-start-column="60" selection-end-line="57" selection-end-column="60" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/model/openhub_Model.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="6681">
<caret line="422" column="70" lean-forward="false" selection-start-line="422" selection-start-column="70" selection-end-line="422" selection-end-column="70" />
<folding>
<element signature="e#7633#7634#0" expanded="true" />
<element signature="e#7656#7657#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/one/Application.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="463">
<caret line="252" column="5" lean-forward="true" selection-start-line="252" selection-start-column="5" selection-end-line="252" selection-end-column="5" />
<state relative-caret-position="17">
<caret line="71" column="31" lean-forward="false" selection-start-line="71" selection-start-column="31" selection-end-line="71" selection-end-column="31" />
<folding>
<element signature="e#1153#1169#0" expanded="true" />
<element signature="e#1381#1397#0" expanded="true" />
<element signature="e#2807#2815#0" expanded="true" />
</folding>
</state>
</provider>

2
new_osseanextractor/osseanextractor.iml
View File

@ -5,13 +5,13 @@
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: junit:junit-dep:4.10" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.1" level="project" />
<orderEntry type="library" name="Maven: us.codecraft:webmagic-core:0.5.2" level="project" />

17
new_osseanextractor/src/main/java/core/ModelPageProcessor.java
View File

@ -2,7 +2,13 @@ package core;
import java.util.ArrayList;
import java.util.List;
/**
*
* <p>Title:站点页面模板生成进程</p>
* <p>Description:生成站点页面模板</p>
* @author liuyizhou
* @date 2016年12月23日
*/
public class ModelPageProcessor implements PageProcessor {
private List<PageModelExtractor> pageModelExtractorList = new ArrayList<PageModelExtractor>();
@ -14,6 +20,11 @@ public class ModelPageProcessor implements PageProcessor {
return modelPageProcessor;
}
/**
* 装载 站点页面模板生成器 列表
* @param clazz 站点模板类对象
* @return
*/
public ModelPageProcessor addPageModel(Class<?> clazz) {
PageModelExtractor pageModelExtractor = PageModelExtractor
.create(clazz);
@ -21,6 +32,10 @@ public class ModelPageProcessor implements PageProcessor {
return this;
}
/**
*
* @param page
*/
public void process(Page page) {
// TODO Auto-generated method stub
for (PageModelExtractor pageModelExtractor : pageModelExtractorList) {

52
new_osseanextractor/src/main/java/core/ModelPipeline4Update.java Normal file
View File

@ -0,0 +1,52 @@
package core;
import java.lang.annotation.Annotation;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.annotation.ExtractBy;
/**
* Created by LouAnt on 2017/2/14 0014
* Description：更新Openhub社区Homepage专用
*/
public class ModelPipeline4Update implements Pipeline{
@SuppressWarnings("rawtypes")
private Map<Class, PageModelPipeline4Update> pageModelPipelines = new ConcurrentHashMap<Class, PageModelPipeline4Update>();
public ModelPipeline4Update() {
}
public ModelPipeline4Update put(Class<?> clazz,
PageModelPipeline4Update<?> pageModelPipeline) {
pageModelPipelines.put(clazz, pageModelPipeline);
return this;
}
@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" })
public void process(ResultItems resultItems,Task task){}
public void processUpdate(ResultItems resultItems,String url, Task task){
for(Map.Entry<Class,PageModelPipeline4Update>classPageModelPipelineEntry : pageModelPipelines.entrySet()){
boolean isSkip = resultItems.getFieldSkip(classPageModelPipelineEntry.getKey().getCanonicalName());
if(isSkip)
continue;
Object o = resultItems.get(classPageModelPipelineEntry.getKey().getCanonicalName());
if(o != null){
Annotation annotation = classPageModelPipelineEntry.getKey().getAnnotation(ExtractBy.class);
if(annotation == null || !((ExtractBy) annotation).multi()){
classPageModelPipelineEntry.getValue().processUpdate(o,url,task);
}else{
List<Object> list = (List<Object>)o;
for(Object o1 : list){
classPageModelPipelineEntry.getValue().processUpdate(o1,url,task);
}
}
}
}
}
}

4
new_osseanextractor/src/main/java/core/Page.java
View File

@ -8,9 +8,7 @@ import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.utils.UrlUtils;
/**
*
* @author Administrator
*
* 单页面实体
*/
public class Page {
private ResultItems resultItems = new ResultItems();

30
new_osseanextractor/src/main/java/core/PageModelExtractor.java
View File

@ -21,34 +21,59 @@ import us.codecraft.webmagic.selector.XpathSelector;
import us.codecraft.webmagic.utils.ClassUtils;
import us.codecraft.webmagic.utils.ExtractorUtils;
/**
*
* <p>Title:页面模板抽取器</p>
* <p>Description:抽取页面模板</p>
* @author liuyizhou
* @date 2016年12月23日
*/
public class PageModelExtractor {
private Class<?> clazz;
private Extractor objectExtractor;
private List<FieldExtractor> fieldExtractors;
private Logger logger = LoggerFactory.getLogger(getClass());
/**
* 创建页面模板
* @param clazz
* @return
*/
public static PageModelExtractor create(Class<?> clazz) {
PageModelExtractor pageModelExtractor = new PageModelExtractor();
pageModelExtractor.init(clazz);
return pageModelExtractor;
}
//初始化页面模板
private void init(Class<?> clazz) {
this.clazz = clazz;
//初始化类对象抽取器objectExtractor
initClassExtractors();
//文件抽取器列表
fieldExtractors = new ArrayList<FieldExtractor>();
//遍历实体类的属性
for (Field field : ClassUtils.getFieldsIncludeSuperClass(clazz)) {
//设置实体类的属性可访问
field.setAccessible(true);
//获取实体类的对应属性的方法
FieldExtractor fieldExtractor = getAnnotationExtractBy(clazz, field);
if (fieldExtractor != null) {
//检查页面模板文件格式
checkFormat(field, fieldExtractor);
fieldExtractors.add(fieldExtractor);
}
}
}
/**
* 检查页面模板格式
* @param field
* @param fieldExtractor
*/
private void checkFormat(Field field, FieldExtractor fieldExtractor) {
// TODO Auto-generated method stub
//获取Field对象注解
Formatter formatter = field.getAnnotation(Formatter.class);
if (formatter != null
&& !formatter.formatter().equals(ObjectFormatter.class)) {
@ -151,6 +176,9 @@ public class PageModelExtractor {
}
}
/**
*初始化类对象抽取器
*/
@SuppressWarnings("deprecation")
private void initClassExtractors() {
// TODO Auto-generated method stub

11
new_osseanextractor/src/main/java/core/PageModelPipeline4Update.java Normal file
View File

@ -0,0 +1,11 @@
package core;
import us.codecraft.webmagic.Task;
/**
* Created by LouAnt on 2017/2/14 0014
* Description：
*/
public interface PageModelPipeline4Update<T> {
public void processUpdate(T t,String url, Task task);
}

16
new_osseanextractor/src/main/java/extension/RawPage.java
View File

@ -7,6 +7,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import core.Page;
/**
* <p>Title:站点详情实体类</p>
* <p>Description:爬取到的站点页面HTML详情实体类</p>
* @author liuyizhou
* @date 2016年12月23日
*/
public class RawPage {
private int id;
@ -29,17 +35,15 @@ public class RawPage {
this.page = new Page();
if (StringUtils.isNotBlank(this.url)) {
if (this.html != null && this.html.length() > 0) {
page.setPageUrl(this.url);
page.setSkip(false);
page.setRawText(this.html);
page.setSkip(false);
page.setTime(crawledTime);
} else {
page.setPageUrl(this.url);
page.setTime(crawledTime);
} else {
page.setSkip(true);
logger.warn("Warnning:" + this.toString()
+ " RawText Is Null OR Empty!");
}
page.setPageUrl(this.url);
page.setTime(crawledTime);
} else {
page.setSkip(true);
logger.warn("Warnning:+" + this.toString()

19
new_osseanextractor/src/main/java/net/trustie/dao/OpenHubProject4Update_Dao.java Normal file
View File

@ -0,0 +1,19 @@
package net.trustie.dao;
import net.trustie.model.openhub_Model;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
/**
* Created by LouAnt on 2017/2/14 0014
* Description：
*/
public interface OpenHubProject4Update_Dao {
/** 根据url更新Homepage*/
@Update("update openhub_project set `homepage`=#{homepage} where url=#{url} ")
public void updateHomepage(@Param("homepage") String homepage,
@Param("url") String url);
}

205
new_osseanextractor/src/main/java/net/trustie/model/openhub_Model.java
View File

@ -4,10 +4,10 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import net.trustie.utils.DateHandler;
import net.trustie.utils.ExtractMutilLink4Openhub;
import net.trustie.utils.Seperator;
import net.trustie.utils.StringHandler;
@ -22,18 +22,6 @@ import core.AfterExtractor;
import core.Page;
import core.ValidateExtractor;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.xsoup.Xsoup;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
@ExtractBy("//div[@id='projects_show_page']")
public class openhub_Model implements AfterExtractor, ValidateExtractor {
@ -49,11 +37,26 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
private static String lastCommitTimeHeader = "most recent commit";
/////////////////////////////
//@ExtractBy("//*[@id='page_contents']/div[3]/div[2]/div/dl/dd[1]/a[1]/@href")
//@ExtractBy("//*[@id="page_contents"]//a[@itemprop="url"]/@href")
@ExtractBy("//div/regex('.*[^a-zA-Z]Homepage[^a-zA-Z].*')")
private String homepage = "";
//@ExtractBy("//*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/dl/dd[1]/a[1]/@href")
//@ExtractBy("//*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/dl/dd[2]/a[1]/@href " +//有Organization的单链接
// "| //*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/dl/dd[1]/a[1]/@href" ) //无Organization的单链接
// @ExtractBy(value="//a/regex(\"<a href='(.*?)'>.*Homepage</a>\",1)")
//@ExtractBy("//*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/dl/dd[2]/a[@itemprop='url']/@href " +
// "| //*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/dl/dd[1]/a[@itemprop='url']/@href " +
// "| //*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/*/*/a/regex(\"<a href='(.*?)'>.*Homepage</a>\",1)/@href" //jquery multi links
// +"| //*[@id='projects_show_page']/div[2]/div[4]/div[2]/div/*/*/a/regex(\"<a href='(.*?)'>.*Homepage</a>\",1)/@href "
//)
@ExtractBy(value="//a/regex(\"<a.*href=\"(.*?)\".*Homepage.*</a>\",1) " )
/*+
" | //a/regex(\"<a.*href=\"(.*?)\".*Download.*</a>\",1)" +
" | //a/regex(\"<a.*href=\"(.*?)\".*Other.*</a>\",1)")
*/
private List<String> homepages = new ArrayList<String>();
private static String homepage ="";
///////////////////////////////////
@ -157,6 +160,12 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
* just for test
*/
private List<String> getHomepages(){
return homepages;
}
public void setHomepages(List<String> homepages){
this.homepages = homepages;
}
private String ansTmp = null;
public String getAnsTmp() {
@ -189,15 +198,8 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
private String urlMd5;
private String source = "openhub";
private int rateNum;
public void afterProcess(Page page) {
//此时的homepage是包含主页链接的整个a标签。需要进行进一步的处理。
homepage = cleanHomepage(homepage);
System.out.println(homepage);
this.urlMd5 = DigestUtils.md5Hex(page.getPageUrl());
SimpleDateFormat bartDateFormat = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss");
@ -424,10 +426,11 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
this.rateLevel = this.rateLevel.substring(0, this.rateLevel.indexOf("/"));
if (StringHandler.isAtLeastOneBlank(this.name, this.activity
/*,this.
description
/*
,this.homepage
,this.description
*/
/*
,getHomepage()
*/
/* ,
* this.licenses
@ -776,137 +779,13 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
/////////////
public String getHomepage() {
return homepage;
}
/*
/抽取a标签的主页链接(即目标a标签下href后面跟的链接)，返回的可能是homepage的真实链接或是一个相对链接（此时有多个homepage）
*/
public String getHref(String homepage) {
String link = "";
String patternString = "\\s*(?i)href\\s*=\\s*(\"([^\"]*\")|'[^']*'|([^'\">\\s]+))";
Pattern pattern = Pattern.compile(patternString,Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(homepage);
while (matcher.find()) {
link=matcher.group().replaceAll("href\\s*=\\s*(['|\"]*)", "").replaceAll("['|\"]", "");
// link=link.replaceAll("href\\s*=\\s*(['|\"]*)", "");
// link=link.replaceAll("['|\"]", "");
}
return link;
}
/*
/判断link是正常的主页链接，还是需要继续处理的相对链接。若是正常目标链接则返回true，若是相对链接则返回false
*/
public Boolean matchLink(String link) {
String patternTrueString = "http.*";
// String patternFalseString = "/p.*";
Pattern patternTrue = Pattern.compile(patternTrueString);
// Pattern patternFalse = Pattern.compile(patternFalseString);
Matcher trueMatcher = patternTrue.matcher(link);
if(trueMatcher.find()) {
return true;
} else {
return false;
}
}
/**
* 抓取某个网页的源代码
* @param urlstr 要抓取网页的地址
* @param charset 网页所使用的编码 如"utf-8","gbk"
* @return 网页源码
* @throws IOException
*/
public String fetchHtml(String urlstr, String charset) {
String result = "";
try {
URL url = new URL(urlstr);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
InputStream is = con.getInputStream();
InputStreamReader isr = new InputStreamReader(is, charset);
int read;
while ((read = isr.read()) != -1) {
result += (char) read;
}
isr.close();
} catch(IOException e) {
e.printStackTrace();
}
return result;
}
/*
/处理双主页情况:获取真实的主页地址，并去重
*/
public String dealWithDoubleHomepage(String link) {
String homePageLink = "";
String homepageHtml = fetchHtml(link,"UTF-8");
Document doc = Jsoup.parse(homepageHtml);
List<String> list = Xsoup
.compile("//*[@id=\"page_contents\"]/div/div/div[1]//a[@target=\"_blank\"]/@href")
.evaluate(doc)
.list();
//去重
HashSet h = new HashSet(list);
list.clear();
list.addAll(h);
for(String each : list) {
homePageLink = (homePageLink+" "+each).trim();
}
homePageLink = homePageLink.replace(" ",";");
return homePageLink;
}
/*
处理带homepage链接的a标签使之得到目标url的总接口方法
*/
public String cleanHomepage(String homepage) {
if(homepage != "") {
String link = "";
link = getHref(homepage);
if(matchLink(link)) {
link = link.trim();
return link;
} else {
String head = "https://www.openhub.net";
link = (head + link).replace(" ","").trim();
link = dealWithDoubleHomepage(link);
return link;
}
} else {
return null;
}
// return homepage;
return getHomepages().isEmpty() ? "" : getValidHomepage(getHomepages().get(0));
}
public void setHomepage(String homepage) {
// String link;
// if(homepage != null) {
// link = getHref(homepage);
// if(matchLink(link)) {
// homepage = link;
// } else {
// String head = "https://www.openhub.net";
// link = (head + link).replace(" ","");
// homepage = link;
// //此处需要后续处理：将此页面爬取下来，并进一步解析页面中的homepage地址（有多个，并且可能需要去重）
// }
// }
this.homepage = homepage;
// System.out.println(this.homepage);
// this.homepage = homepage;
this.homepage = getHomepages().isEmpty() ? "" : getHomepages().get(0);
///////////////////////
}
@ -1397,11 +1276,15 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
public void setHistory(int history) {
this.history = history;
}
// public List<String> getLinkInfos() {
// return linkInfos;
// }
//
// public void setLinkInfos(List<String> linkInfos) {
// this.linkInfos = linkInfos;
// }
public String getValidHomepage(String homePage){
String result = homePage;
//目前的Homepage抽取规则，以“/”开头的如"/p/jQuery"均是多链接的情形
if(!homePage.equals("") && homePage.startsWith("/")){
//获取Homepage列表中的Homepage
result = new ExtractMutilLink4Openhub().extractLinks(homePage);
}
return result;
}
}

22
new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
View File

@ -20,13 +20,17 @@ import core.Page;
import core.ValidateExtractor;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ExtractBy("//body[@id='pg_project']/div[@id='page-body']/article[@id='project']")
// 限定抽取区域
public class sourceforge_Model implements AfterExtractor, ValidateExtractor{
///////////////
@ExtractBy("//a[@id='homepage']/@href")
@ExtractBy("//a[@id='homepage']/@href " +
" | //*[@class='homepage-link']/a/@href")
private String homepage;
//////////////////
@ -609,4 +613,20 @@ public void setHomepage(String homepage) {
+ ", pageMD5=" + pageMd5 + ", history=" + history + "]";
}
/**
* 正则验证homepage的有效性
* @param homepage
* @return
*/
private String checkHomepage(String homepage){
String newHomepage = homepage;
String key = "^([hH][tT]{2}[pP]://|[hH][tT]{2}[pP][sS]://)(([a-zA-Z0-9\\._-]+\\.[a-zA-Z]{2,6})|([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}))(:[0-9]{1,4})*(/[a-zA-Z0-9\\&%_\\./-~-]*)?";
Pattern pattern = Pattern.compile(key,Pattern.CASE_INSENSITIVE);
//Homepage无效则使用页面url
if(null == homepage || "".equals(homepage) || !pattern.matcher(newHomepage).matches()){
newHomepage = this.getUrl();
}
return newHomepage;
}
}

184
new_osseanextractor/src/main/java/net/trustie/one/Application.java
View File

@ -18,6 +18,7 @@ import net.trustie.dao.RecordDao;
import net.trustie.downloader.DataBasePageErrorOutPut;
import net.trustie.downloader.PageDao;
import net.trustie.model.cto51_blog_Model;
import net.trustie.utils.Constant;
import net.trustie.utils.DateHandler;
import org.springframework.beans.BeansException;
@ -30,13 +31,22 @@ import core.ModelPipeline;
import core.PageModelPipeline;
import extension.RawPage;
/**
* <p>Title:抽取服务程序入口</p>
* <p>Description:利用抽取模板，从爬虫数据库中抽取站点的项目、文档信息，
* 抽取至抽取数据库中对应站点的分表中</p>
* @author
* @date 2016年12月22日
*/
@Component("Application")
public class Application {
private String sitesPath = "tasks.txt";
//indicate if a site is being extracted
/** 站点列表配置文件路径 */
private String sitesPath = Constant.SITE_PATH;
/** 抽取状态：indicate if a site is being extracted ，状态为true时表示正在抽取中无需重复抽取*/
public static Map<String,Boolean> extractState = new HashMap<String,Boolean>();
private ExecutorService pool = Executors.newFixedThreadPool(20);
/** 线程池，用以装载抽取线程 参数：20为最大线程数 */
private ExecutorService pool = Executors.newFixedThreadPool(Constant.POOL_MAX_SIZE);
public static void main(String args[]){
((Application)AppContext.appContext.getBean("Application")).startJob();
}
@ -44,23 +54,23 @@ public class Application {
public void startJob() {
while(true){
//从文件中读取要抽取哪些站点
List<String> sites = readSites();
List<String> sites = readSitesFromConfig();
if(sites.size() == 0)
continue;
//看site被抽取过没，state为true说明已经抽取过了
for( String site : sites){
Boolean state = extractState.get(site);
if( state == null || state == false){
extractState.put(site, true);
}
else if(state == true){
else if(state == true){//抽取中，无需重复抽取
continue;
}
//用线程池抽取各个站点
ExtractThread et = (ExtractThread)AppContext.appContext.getBean("extractThread");
et.setSite(site);
pool.execute(et);
//提交任务到线程池，无返回值
pool.execute(et);
}
try {
System.out.println(Thread.currentThread().getName() + ": " +
@ -76,7 +86,11 @@ public class Application {
}
private List<String> readSites() {
/**
* 读取站点配置文件
* @return 站点列表
*/
private List<String> readSitesFromConfig() {
File file = new File(sitesPath);
LinkedList<String> tasks = new LinkedList<String>();
BufferedReader reader = null;
@ -101,155 +115,5 @@ public class Application {
}
}
@Component("extractThread")
@Scope("prototype")
class ExtractThread implements Runnable{
private String site;
private static final int BatchExtractSize = 100;
public List<String> modelName = new ArrayList<String>();
private Class pageModel;
@Resource
private PageDao pagedao;
@Qualifier("errorPageToDB")
@Autowired
private DataBasePageErrorOutPut pageErrorOutPut;
@Resource
private RecordDao recordDao;
public ExtractThread(){
}
public ExtractThread(String site){
this.site = site;
}
public void setSite(String site){
this.site = site;
}
public void run() {
//设置threadname 以便查看输出日志
Thread.currentThread().setName(site + "_thread");
this.pageErrorOutPut.setTableName(site + "_error_page");
try {
System.out.println(Thread.currentThread().getName() + ": " +
"extract begain" + " @ " + DateHandler.getExtractTime());
if(site.equals("51cto_blog"))
pageModel = cto51_blog_Model.class;
else
pageModel = Class.forName("net.trustie.model." + site +"_Model");
modelName.add(pageModel.getCanonicalName());
//获取该站点的抽取模板和要抽取的html页面
int lastId = getLastId(site);
List<RawPage> pages = getPages(site,lastId);
Extractor extractor = new Extractor();
RawPage result = null;
while(pages.size() > 0){
for( RawPage page : pages){
try{
result = extractor.extract(page,pageModel);
//持久化 并 更新抽取历史
saveResult(site,result);
}catch (Exception e){
e.printStackTrace();
pageErrorOutPut.returnErrorPage(page, e);
}
}
updateLastId(site,lastId + pages.size());
lastId = getLastId(site);
pages = getPages(site,lastId);
}
Application.extractState.put(site, false);
System.out.println(Thread.currentThread().getName() + ": " +
"extract over, id=" + lastId + " @ " + DateHandler.getExtractTime());
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch(Exception e){
System.out.println(e.getMessage());
e.printStackTrace();
System.out.println(Thread.currentThread().getName() + ": " +" 发生异常 ");
}
}
private void saveResult(String site, RawPage rawPage ) {
ModelPipeline pl = new ModelPipeline();
try {
pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
} catch (BeansException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (!rawPage.getPage().getResultItems().isSkip()) {
pl.process(rawPage.getPage().getResultItems(), null);
}
if (!rawPage.getPage()
.isAllResultSkip(
modelName.toArray(new String[modelName
.size()]))) {
rawPage.setStored(true);
}else{
}
if (!rawPage.isExtracted() || !rawPage.isStored())
pageErrorOutPut.returnErrorPage(rawPage,
"May Caused By Model Problem! Or Page Error!");
}
private void updateLastId(String site, int currentId) {
recordDao.updateRecord(site,currentId);
}
private void saveResult(String site, List<RawPage> results) {
//System.out.println("save result");
ModelPipeline pl = new ModelPipeline();
try {
pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
} catch (BeansException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
for(RawPage rawPage : results){
if (!rawPage.getPage().getResultItems().isSkip()) {
pl.process(rawPage.getPage().getResultItems(), null);
}
if (!rawPage.getPage()
.isAllResultSkip(
modelName.toArray(new String[modelName
.size()]))) {
rawPage.setStored(true);
}else{
System.out.println(rawPage.getPage().getResultItems()+"*****************************");
}
if (!rawPage.isExtracted() || !rawPage.isStored())
pageErrorOutPut.returnErrorPage(rawPage,
"May Caused By Model Problem! Or Page Error!");
}
}
private List<RawPage> getPages(String site, int lastId) {
LinkedList<RawPage> pages = new LinkedList<RawPage>();
pages = pagedao.getDetailPages(site + "_html_detail",lastId,BatchExtractSize);
return pages;
}
private int getLastId(String site) {
Integer lastRecord = recordDao.getLastRecord(site);
if(lastRecord == null){
lastRecord = pagedao.getMinId(site + "_html_detail") - 1;
recordDao.insertRecord(site,lastRecord);
}
return lastRecord;
}
}

206
new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java Normal file
View File

@ -0,0 +1,206 @@
package net.trustie.one;
import core.ModelPipeline;
import core.PageModelPipeline;
import extension.RawPage;
import net.trustie.dao.RecordDao;
import net.trustie.downloader.DataBasePageErrorOutPut;
import net.trustie.downloader.PageDao;
import net.trustie.model.cto51_blog_Model;
import net.trustie.utils.Constant;
import net.trustie.utils.DateHandler;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
/**
* <p>Title:抽取线程</p>
* <p>Description:抽取项目的任务主线程</p>
* @author liuyizhou
* @date 2016年12月22日
*/
@Component("extractThread")
@Scope("prototype")
public class ExtractThread implements Runnable{
/** 待抽取的站点名称 */
private String site;
/** 待抽取结果条数限制 */
private static final int BatchExtractSize = Constant.BATCH_EXTRACT_SIZE;
/** 模板名称列表 */
public List<String> modelName = new ArrayList<String>();
/** 动态生成站点模板 */
private Class pageModel;
@Resource
private PageDao pagedao;
/** 输出错误页面 */
@Qualifier("errorPageToDB")
@Autowired
private DataBasePageErrorOutPut pageErrorOutPut;
/** records表操作接口实例化 */
@Resource
private RecordDao recordDao;
public ExtractThread(){
}
public ExtractThread(String site){
setSite(site);
}
public void setSite(String site){
this.site = site;
}
public void run() {
//设置thread name 以便查看输出日志
Thread.currentThread().setName(site + "_thread");
this.pageErrorOutPut.setTableName(site + "_error_page");
try {
System.out.println(Thread.currentThread().getName() + ": " +
"extract begain" + " @ " + DateHandler.getExtractTime());
if(site.equals("51cto_blog"))
pageModel = cto51_blog_Model.class;
else
pageModel = Class.forName("net.trustie.model." + site +"_Model");
modelName.add(pageModel.getCanonicalName());
//获取该站点的抽取模板和要抽取的html页面
int lastId = getLastId(site);
List<RawPage> pages = getPages(site,lastId);
Extractor extractor = new Extractor();
RawPage result = null;
while(pages.size() > 0){
for( RawPage page : pages){
try{
result = extractor.extract(page,pageModel);
//持久化 并 更新抽取历史
saveResult(site,result);
}catch (Exception e){
e.printStackTrace();
pageErrorOutPut.returnErrorPage(page, e);
}
}
updateLastId(site,lastId + pages.size());
lastId = getLastId(site);
pages = getPages(site,lastId);
}
Application.extractState.put(site, false);
System.out.println(Thread.currentThread().getName() + ": " +
"extract over, id=" + lastId + " @ " + DateHandler.getExtractTime());
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch(Exception e){
System.out.println(e.getMessage());
e.printStackTrace();
System.out.println(Thread.currentThread().getName() + ": " +" 发生异常 ");
}
}
private void saveResult(String site, RawPage rawPage ) {
ModelPipeline pl = new ModelPipeline();
try {
pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
} catch (BeansException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (!rawPage.getPage().getResultItems().isSkip()) {
pl.process(rawPage.getPage().getResultItems(), null);
}
if (!rawPage.getPage()
.isAllResultSkip(
modelName.toArray(new String[modelName
.size()]))) {
rawPage.setStored(true);
}else{
}
if (!rawPage.isExtracted() || !rawPage.isStored())
pageErrorOutPut.returnErrorPage(rawPage,
"May Caused By Model Problem! Or Page Error!");
}
private void updateLastId(String site, int currentId) {
recordDao.updateRecord(site,currentId);
}
private void saveResult(String site, List<RawPage> results) {
//System.out.println("save result");
ModelPipeline pl = new ModelPipeline();
try {
pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
} catch (BeansException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
for(RawPage rawPage : results){
if (!rawPage.getPage().getResultItems().isSkip()) {
pl.process(rawPage.getPage().getResultItems(), null);
}
if (!rawPage.getPage()
.isAllResultSkip(
modelName.toArray(new String[modelName
.size()]))) {
rawPage.setStored(true);
}else{
System.out.println(rawPage.getPage().getResultItems()+"*****************************");
}
if (!rawPage.isExtracted() || !rawPage.isStored())
pageErrorOutPut.returnErrorPage(rawPage,
"May Caused By Model Problem! Or Page Error!");
}
}
/**
* 获取站点site_html_detail表中，id大于lastId的 BatchExtractSize 条记录
* @param site
* @param lastId
* @return
*/
private List<RawPage> getPages(String site, int lastId) {
LinkedList<RawPage> pages = new LinkedList<RawPage>();
pages = pagedao.getDetailPages(site + "_html_detail",lastId,BatchExtractSize);
return pages;
}
/**
* 获取站点site的最后id
* @param site
* @return
*/
private int getLastId(String site) {
/** 从records表中，获取最新的抽取记录的id */
Integer lastRecord = recordDao.getLastRecord(site);
if(lastRecord == null){//最新抽取记录为空，则新增
//从对应站点HTML详情表site_html_detail中，获取最新的id
lastRecord = pagedao.getMinId(site + "_html_detail") - 1;//TODO 为什么是min(ID)
//最新id插入records表
recordDao.insertRecord(site,lastRecord);
}
return lastRecord;
}
}

4
new_osseanextractor/src/main/java/net/trustie/one/Extractor.java
View File

@ -10,6 +10,10 @@ import core.ModelPageProcessor;
import core.Pipeline;
import extension.RawPage;
/**
* 抽取核心类
* 主要执行
*/
public class Extractor {
public List<RawPage> extract(List<RawPage> pages, Class<?> modelClass ) {

163
new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java Normal file
View File

@ -0,0 +1,163 @@
package net.trustie.one;
import core.ModelPipeline;
import core.ModelPipeline4Update;
import core.PageModelPipeline;
import core.PageModelPipeline4Update;
import extension.RawPage;
import net.trustie.dao.RecordDao;
import net.trustie.downloader.DataBasePageErrorOutPut;
import net.trustie.downloader.PageDao;
import net.trustie.utils.Constant;
import net.trustie.utils.DateHandler;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* Created by LouAnt on 2017/2/13 0013
* Description：Openhub 社区项目Homepage字段重新抽取
*/
@Component("ReExtractor")
public class OpenhubReExtractor {
/** 抽取状态：indicate if a site is being extracted ，状态为true时表示正在抽取中无需重复抽取*/
public static Map<String,Boolean> extractState = new HashMap<String,Boolean>();
/** 线程池，用以装载抽取线程 参数：20为最大线程数 */
private ExecutorService pool = Executors.newFixedThreadPool(Constant.POOL_MAX_SIZE);
public static void main(String[] args){
((OpenhubReExtractor) AppContext.appContext.getBean("ReExtractor")).start();
}
public void start(){
while(true){
ReExtractThread reExtractThread = (ReExtractThread)AppContext.appContext.getBean("reExtractThread");
pool.execute(reExtractThread);
}
}
}
/**
* 重新抽取的主线程
*/
@Component("reExtractThread")
@Scope("prototype")
class ReExtractThread implements Runnable{
/** 待抽取的站点名称 */
private String site;
/** 待抽取结果条数限制 */
private static final int BatchExtractSize = Constant.BATCH_EXTRACT_SIZE;
/** 模板名称列表 */
public List<String> modelName = new ArrayList<String>();
/** 动态生成站点模板 */
private Class pageModel;
@Resource
private PageDao pagedao;
/** 输出错误页面 */
@Qualifier("errorPageToDB")
@Autowired
private DataBasePageErrorOutPut pageErrorOutPut;
/** records表操作接口实例化 */
@Resource
private RecordDao recordDao;
public void run(){
/**站点名称*/
String site = "openhub";
Thread.currentThread().setName(site + "reextractor_thread");
this.pageErrorOutPut.setTableName(site+"_error_page");
try {
pageModel = Class.forName("net.trustie.model."+site+"_Model");
modelName.add(pageModel.getCanonicalName());
int lastId = getLastId(site);
List<RawPage> pages = getPages(site,lastId);
Extractor extractor = new Extractor();
RawPage rawPage = null;
while(pages.size() > 0){
for(RawPage page : pages){
try {
rawPage = extractor.extract(page, pageModel);
saveResult(site, rawPage);
}catch (Exception e){
e.printStackTrace();
// pageErrorOutPut.returnErrorPage(page, e);错误页面
}
}
//更新抽取游标
updateLastId(site,lastId + pages.size());
lastId = getLastId(site);
pages = getPages(site,lastId);
}
Application.extractState.put(site, false);
System.out.println(Thread.currentThread().getName() + ": " +
"extract over, id=" + lastId + " @ " + DateHandler.getExtractTime());
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch(Exception e){
System.out.println(e.getMessage());
e.printStackTrace();
System.out.println(Thread.currentThread().getName() + ": " +" 发生异常 ");
}
}
private void saveResult(String site,RawPage rawPage){
ModelPipeline4Update pl = new ModelPipeline4Update();
try {
pl.put(pageModel, (PageModelPipeline4Update) AppContext.appContext.getBean(site + "_pipeline_4update"));
}catch (BeansException e){
e.printStackTrace();
}
if(!rawPage.getPage().getResultItems().isSkip()){
pl.processUpdate(rawPage.getPage().getResultItems(),rawPage.getUrl(),null);
}
}
/**
* 获取站点site_html_detail表中，id大于lastId的 BatchExtractSize 条记录
* @param site
* @param lastId
* @return
*/
private List<RawPage> getPages(String site, int lastId) {
LinkedList<RawPage> pages = new LinkedList<RawPage>();
pages = pagedao.getDetailPages(site + "_html_detail",lastId,BatchExtractSize);
return pages;
}
/**
* 获取站点site的最后id
* @param site
* @return
*/
private int getLastId(String site) {
/** 从records表中，获取最新的抽取记录的id */
Integer lastRecord = recordDao.getLastRecord(site+"_reextractor");
if(lastRecord == null){//最新抽取记录为空，则新增
//从对应站点HTML详情表site_html_detail中，获取最新的id
lastRecord = pagedao.getMinId(site + "_html_detail") - 1;
//最新id插入records表
recordDao.insertRecord(site+"_reextractor",lastRecord);
}
return lastRecord;
}
private void updateLastId(String site, int currentId) {
recordDao.updateRecord(site+"_reextractor",currentId);
}
}

34
new_osseanextractor/src/main/java/net/trustie/pipeline/OpenHubProject_pipeline_4update.java Normal file
View File

@ -0,0 +1,34 @@
package net.trustie.pipeline;
import core.PageModelPipeline4Update;
import net.trustie.dao.OpenHubProject4Update_Dao;
import net.trustie.model.openhub_Model;
import us.codecraft.webmagic.Task;
import javax.annotation.Resource;
import org.springframework.stereotype.Component;
import net.trustie.dao.OpenHubProject_Dao;
import net.trustie.model.openhub_Model;
import core.PageModelPipeline;
/**
* Created by LouAnt on 2017/2/14 0014
* Description：
*/
@Component("openhub_pipeline_4update")
public class OpenHubProject_pipeline_4update implements PageModelPipeline4Update<openhub_Model> {
@Resource
private OpenHubProject4Update_Dao openHubProject4Update_dao;
/**
* 更新Homepage专用
* @param openhub_model
* @param task
*/
@Override
public void processUpdate(openhub_Model openhub_model, String url, Task task){
openHubProject4Update_dao.updateHomepage(openhub_model.getHomepage(),url);
}
}

18
new_osseanextractor/src/main/java/net/trustie/utils/Constant.java Normal file
View File

@ -0,0 +1,18 @@
package net.trustie.utils;
/**
* <p>Title:公共变量类</p>
* <p>Description:用于存放全局静态变量</p>
* @author liuyizhou
* @date 2016年12月22日
*/
public class Constant {
/** 程序根目录*/
private final static String CURRENT_PATH = System.getProperty("user.dir");
/** 开源站点列表配置文件路径 */
public final static String SITE_PATH = CURRENT_PATH + "\\tasks.txt";
/** 线程池最大线程数 */
public final static int POOL_MAX_SIZE = 20;
/** 抽取站点结果时，待抽取结果条数限制*/
public final static int BATCH_EXTRACT_SIZE = 100;
}

100
new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java Normal file
View File

@ -0,0 +1,100 @@
package net.trustie.utils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List;
/**
* Created by LouAnt on 2017/2/10 0010
* Description：抽取openhub中多链接的项目
*/
public class ExtractMutilLink4Openhub implements PageProcessor{
private Site site = Site.me().setRetryTimes(5).setSleepTime(500);
private static List<String> homepages;
private static Page _page;
static ExtractMutilLink4Openhub extractMutilLink4Openhub = new ExtractMutilLink4Openhub();
@Override
public void process(Page page) {
//抽取Homepage列表，Html格式参考 https://www.openhub.net/p/zimbra/links
ExtractMutilLink4Openhub.homepages = page.getHtml().xpath("//div[@class='links center-block']/div[1]/div[2]/div/h5/a").links().all();
}
@Override
public Site getSite() {
return site;
}
public List<String> getHomepages(String url){
try{
int count = 5;
//为防止一次抽取不成功，抽取5次
while(ExtractMutilLink4Openhub.homepages==null && count > 0 ) {
ExtractMutilLink4Openhub.doExtract(url);
count--;
}
}catch(Exception e) {
}
return ExtractMutilLink4Openhub.homepages;
}
/**
* 获取多个link，用“;”间隔
* @param url
* @return
*/
public String extractLinks(String url){
String homepage = "";
doExtract(url);
try {
List<String> tempPages = getHomepages(url);
if (tempPages != null && !tempPages.isEmpty()) {
for (String link : tempPages) {
if(!homepage.contains(link)){//避免加入重复的链接
homepage += link + ";";
}
}
homepage = homepage.substring(0, homepage.lastIndexOf(";"));
}
}catch (Exception e){
e.printStackTrace();
}
return homepage;
}
/**
* 获取第一个link作为Homepage
* @param url
* @return
*/
public String extractLink(String url){
String homepage = url;
doExtract(url);
try {
List<String> tempPages = getHomepages(url);
homepage = (tempPages != null && !tempPages.isEmpty()) ? tempPages.get(0) : url;
}catch (Exception e){
e.printStackTrace();
}
return homepage;
}
/**
* 抽取Homepage核心方法
* @param url 抽取目标url链接
*/
public static void doExtract(String url){
Spider.create(extractMutilLink4Openhub).addUrl(new String("https://www.openhub.net"+url)).thread(1).run();
}
public static void main(String[] args){
String result = new ExtractMutilLink4Openhub().extractLink("/p/jQuery/links");
System.out.println(result);
}
}

16
project_match/bin/start_get_synonyms.sh~HEAD Normal file
View File

@ -0,0 +1,16 @@
#!/bin/bash
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
tmp='./target/classes':$tmp
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
tmp='./bin/resources':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
java -classpath $CLASSPATH com.ossean.GetSynonyms >>log/getsynonyms_2017.log 2>&1 &

20
project_match/bin/start_get_synonyms.sh~liuyizhou Normal file
View File

@ -0,0 +1,20 @@
#!/bin/bash
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
tmp='./target/classes':$tmp
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
tmp='./bin/resources':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH com.ossean.GetSynonyms >>log/getsynonyms_2017.log 2>&1 &