zaihui
/
educoder
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

put the items which has failed to dowanload homepage-page into db(openhub)

This commit is contained in:
wrzzx 2017-02-19 14:23:50 +08:00
parent ef65d3e927
commit c79044416e
11 changed files with 1172 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

496
crawler/moreSmarterCrawler/fetch_networks/.idea/workspace.xml
View File

@ -4,9 +4,13 @@
<list default="true" id="f59140c6-b612-48dd-baf7-0162af9922f0" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/.idea/workspace.xml" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/dao/OpenHubProject_Dao.java" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/dao/OpenHubProject_Dao.java" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/osseanextractor.iml" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/osseanextractor.iml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/pom.xml" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/pom.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/model/openhub_Model.java" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/model/openhub_Model.java" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java" afterPath="$PROJECT_DIR$/../../../new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java" />
</list>
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
@ -33,28 +37,48 @@
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="ListHtmlCrawler.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java">
<file leaf-file-name="DetailHtmlProcessor.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<state relative-caret-position="307">
<caret line="45" column="43" lean-forward="true" selection-start-line="45" selection-start-column="43" selection-end-line="45" selection-end-column="43" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="ListHtmlProcessor.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java">
<file leaf-file-name="DetailHtmlCrawler.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="238">
<caret line="44" column="22" lean-forward="true" selection-start-line="44" selection-start-column="22" selection-end-line="44" selection-end-column="22" />
<state relative-caret-position="-951">
<caret line="131" column="57" lean-forward="true" selection-start-line="131" selection-start-column="57" selection-end-line="131" selection-end-column="57" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DetailHtmlPipeline.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="33" column="57" lean-forward="true" selection-start-line="33" selection-start-column="57" selection-end-line="33" selection-end-column="57" />
<folding>
<element signature="e#4827#4835#0" expanded="true" />
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="DetailHtmlDao.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="123">
<caret line="17" column="26" lean-forward="true" selection-start-line="17" selection-start-column="26" selection-end-line="17" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="Git.Settings">
@ -144,7 +168,6 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="PackagesPane" />
<pane id="ProjectPane">
<subPane>
<PATH>
@ -157,10 +180,241 @@
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="resources" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="webmagic" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="utils" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="webmagic" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="webmagic" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="one" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="webmagic" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="one" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="model" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="webmagic" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="one" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="dao" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="fetchnetworks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="fetch_networks" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="java" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
<pane id="Scope" />
<pane id="Scratches" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
@ -624,11 +878,14 @@
<workItem from="1486695485959" duration="431000" />
<workItem from="1486708709471" duration="13000" />
<workItem from="1487174066127" duration="4746000" />
<workItem from="1487307404778" duration="352000" />
<workItem from="1487335460841" duration="598000" />
<workItem from="1487407224064" duration="8108000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="13656000" />
<option name="totallyTimeSpent" value="22714000" />
</component>
<component name="TodoView">
<todo-panel id="selected-file">
@ -641,12 +898,11 @@
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1382" height="744" extended-state="6" />
<editor active="false" />
<editor active="true" />
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Nl-Palette" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32761577" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
@ -659,7 +915,7 @@
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.23373677" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24508321" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
@ -671,6 +927,7 @@
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32761577" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
@ -694,7 +951,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -702,9 +961,7 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#4827#4835#0" expanded="true" />
</folding>
<folding />
</state>
</provider>
</entry>
@ -712,7 +969,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -720,8 +979,70 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<folding>
<element signature="e#4827#4835#0" expanded="true" />
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
@ -730,7 +1051,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -738,7 +1061,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -746,7 +1071,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -754,7 +1081,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -762,15 +1091,9 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
@ -786,11 +1109,7 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="imports" expanded="false" />
<element signature="e#3598#3607#0" expanded="false" />
<element signature="e#4491#4499#0" expanded="false" />
</folding>
<folding />
</state>
</provider>
</entry>
@ -802,18 +1121,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="34">
<caret line="36" column="21" lean-forward="false" selection-start-line="36" selection-start-column="21" selection-end-line="36" selection-end-column="21" />
<folding>
<element signature="imports" expanded="false" />
<element signature="e#3598#3607#0" expanded="false" />
<element signature="e#4491#4499#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/spring/applicationContext-myBatis.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
@ -826,19 +1133,100 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
<folding>
<element signature="imports" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/spider.conf.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/main.iml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/spring/applicationContext.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/ListHtmlProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="238">
<caret line="44" column="22" lean-forward="true" selection-start-line="44" selection-start-column="22" selection-end-line="44" selection-end-column="22" />
<state relative-caret-position="17">
<caret line="34" column="0" lean-forward="false" selection-start-line="34" selection-start-column="0" selection-end-line="34" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/model/AbstractHtml.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-17">
<caret line="8" column="32" lean-forward="true" selection-start-line="8" selection-start-column="32" selection-end-line="8" selection-end-column="32" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/model/DetailHtml.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="187">
<caret line="11" column="25" lean-forward="true" selection-start-line="11" selection-start-column="25" selection-end-line="11" selection-end-column="25" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/utils/DBPipeline.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="68">
<caret line="4" column="54" lean-forward="true" selection-start-line="4" selection-start-column="54" selection-end-line="4" selection-end-column="54" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/dao/DetailHtmlDao.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="123">
<caret line="17" column="26" lean-forward="true" selection-start-line="17" selection-start-column="26" selection-end-line="17" selection-end-column="26" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlPipeline.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="255">
<caret line="33" column="57" lean-forward="true" selection-start-line="33" selection-start-column="57" selection-end-line="33" selection-end-column="57" />
<folding>
<element signature="e#4827#4835#0" expanded="true" />
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlProcessor.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="307">
<caret line="45" column="43" lean-forward="true" selection-start-line="45" selection-start-column="43" selection-end-line="45" selection-end-column="43" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/trustie/webmagic/one/DetailHtmlCrawler.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-951">
<caret line="131" column="57" lean-forward="true" selection-start-line="131" selection-start-column="57" selection-end-line="131" selection-end-column="57" />
<folding />
</state>
</provider>
</entry>
</component>
</project>

926
new_osseanextractor/.idea/workspace.xml
View File

File diff suppressed because it is too large Load Diff

1
new_osseanextractor/osseanextractor.iml
View File

@ -11,7 +11,6 @@
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
<orderEntry type="library" name="Maven: junit:junit-dep:4.10" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.1" level="project" />
<orderEntry type="library" name="Maven: us.codecraft:webmagic-core:0.5.2" level="project" />

1
new_osseanextractor/pom.xml
View File

@ -82,6 +82,5 @@
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
</project>

46
new_osseanextractor/src/main/java/net/trustie/dao/OpenHubRetry_Dao.java Normal file
View File

@ -0,0 +1,46 @@
package net.trustie.dao;
import net.trustie.model.openhub_retry_Model;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
/**
* Created by zaihuilvcha on 2017/2/18.
*/
public class OpenHubRetry_Dao {
private Connection conn = null;
private Statement stmt = null;
public OpenHubRetry_Dao() {
try {
Class.forName("com.mysql.jdbc.Driver");
String url = "jdbc:mysql://localhost:3306/extract_result?user=root&password=123456";
conn = DriverManager.getConnection(url);
stmt = conn.createStatement();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException e) {
e.printStackTrace();
}
}
public int add(openhub_retry_Model oprm) {
try {
String sql = "INSERT INTO `extract_result`.`openhub_download_fail` (`url`, `html`) VALUES (?, ?);";
PreparedStatement ps = conn.prepareStatement(sql);
ps.setString(1, oprm.getUrl());
ps.setString(2,oprm.getHtml());
return ps.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
}
return -1;
}
}

56
new_osseanextractor/src/main/java/net/trustie/model/openhub_Model.java
View File

@ -6,7 +6,8 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import net.trustie.utils.DateHandler;
import core.*;
import net.trustie.dao.OpenHubRetry_Dao;
import net.trustie.utils.ExtractMutilLink4Openhub;
import net.trustie.utils.Seperator;
import net.trustie.utils.StringHandler;
@ -18,9 +19,8 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import core.AfterExtractor;
import core.Page;
import core.ValidateExtractor;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import us.codecraft.webmagic.model.annotation.ExtractBy;
@ExtractBy("//div[@id='projects_show_page']")
@ -49,10 +49,11 @@ public class openhub_Model implements AfterExtractor, ValidateExtractor {
// "| //*[@id='projects_show_page']/div[2]/div[3]/div[2]/div/*/*/a/regex(\"<a href='(.*?)'>.*Homepage</a>\",1)/@href" //jquery multi links
// +"| //*[@id='projects_show_page']/div[2]/div[4]/div[2]/div/*/*/a/regex(\"<a href='(.*?)'>.*Homepage</a>\",1)/@href "
//)
@ExtractBy(value="//a/regex(\"<a.*href=\"(.*?)\".*Homepage.*</a>\",1) " +
" | //a/regex(\"<a.*href=\"(.*?)\".*Download.*</a>\",1)" +
" | //a/regex(\"<a.*href=\"(.*?)\".*Other.*</a>\",1)")
private List<String> homepages = new ArrayList<String>();
// @ExtractBy(value="//a/regex(\"<a.*href=\"(.*?)\".*Homepage.*</a>\",1) " +
// " | //a/regex(\"<a.*href=\"(.*?)\".*Download.*</a>\",1)" +
// " | //a/regex(\"<a.*href=\"(.*?)\".*Other.*</a>\",1)")
@ExtractBy(value="//a/regex(\"<a.*href=\"(.*?)\".*Homepage.*</a>\",1) ")
private List<String> homepages = new ArrayList<String>();
private static String homepage ="";
///////////////////////////////////
@ -401,6 +402,7 @@ private List<String> homepages = new ArrayList<String>();
}
@Override
public void validate(Page page) {
//
@ -423,18 +425,43 @@ private List<String> homepages = new ArrayList<String>();
}
this.rateLevel = this.rateLevel.substring(0, this.rateLevel.indexOf("/"));
if (StringHandler.isAtLeastOneBlank(this.name, this.activity
,this.description
,getHomepage()
,this.getHomepage()
/* ,
* this.licenses
*/)) {
page.setResultSkip(this, true);
//多homepage抽取时，下载失败，需要将对应page存库.downloadFailFlag为存库失败标志
if(downloadFailFlag) {
//存库开始，先将下载失败标志置为假
downloadFailFlag = false;
System.out.println("homepage页面下载失败，条目准备入库...");
openhub_retry_Model oprm = new openhub_retry_Model();
oprm.setUrl(page.getPageUrl());
oprm.setHtml(page.getRawText());
OpenHubRetry_Dao opDao = new OpenHubRetry_Dao();
opDao.add(oprm);
System.out.println("下载失败条目已存入数据库...");
}
return;
}
}
private void handleQuickRef(Element quickRef) {
@ -1274,6 +1301,7 @@ private List<String> homepages = new ArrayList<String>();
this.history = history;
}
private boolean downloadFailFlag = false;
public String getValidHomepage(String homePage){
String result = homePage;
@ -1281,6 +1309,16 @@ private List<String> homepages = new ArrayList<String>();
if(!homePage.equals("") && homePage.startsWith("/")){
//获取Homepage列表中的Homepage
result = new ExtractMutilLink4Openhub().extractLinks(homePage);
/**
* result=""时，需要对相应page加入存库处理。
* 这里必然是多homepage情况。所以此处若正常则result不为空，若为空则说明下载失败了，必须存库
*/
if(result.equals("")) {
//存库标志位置为true
downloadFailFlag = true;
}
}
return result;
}

36
new_osseanextractor/src/main/java/net/trustie/model/openhub_retry_Model.java Normal file
View File

@ -0,0 +1,36 @@
package net.trustie.model;
/**
* Created by zaihuilvcha on 2017/2/18.
*/
public class openhub_retry_Model {
String url;
String html;
public void setUrl(String url) {
this.url = url;
}
public void setHtml(String html) {
this.html = html;
}
public String getUrl() {
return url;
}
public String getHtml() {
return html;
}
@Override
public String toString() {
return "openhub_retry_Model{" +
"url='" + url + '\'' +
", html='" + html + '\'' +
'}';
}
}

6
new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
View File

@ -28,9 +28,9 @@ import java.util.regex.Pattern;
public class sourceforge_Model implements AfterExtractor, ValidateExtractor{
///////////////
@ExtractBy("//a[@id='homepage']/@href " +
" | //*[@class='homepage-link']/a/@href")
//@ExtractBy("//a[@id='homepage']/@href " +
// " | //*[@class='homepage-link']/a/@href")
@ExtractBy("//*[@id='homepage']/@href ")
private String homepage;
//////////////////

12
new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java
View File

@ -84,13 +84,16 @@ public class ExtractThread implements Runnable{
Extractor extractor = new Extractor();
RawPage result = null;
while(pages.size() > 0){
/**
* 注意这里的page就是detail表中的一条记录。
*/
for( RawPage page : pages){
try{
long startTime=System.currentTimeMillis(); //获取开始时间
result = extractor.extract(page,pageModel);
long endTime=System.currentTimeMillis(); //获取结束时间
// System.out.println("页面抽取时间： "+(endTime-startTime)+"ms");
// System.out.println("页面抽取时间： "+(endTime-startTime)+"ms");
startTime=System.currentTimeMillis(); //获取开始时间
//持久化 并 更新抽取历史
saveResult(site,result);
@ -98,10 +101,11 @@ public class ExtractThread implements Runnable{
// System.out.println("结果保存时间： "+(endTime-startTime)+"ms");
}catch (Exception e){
e.printStackTrace();
// pageErrorOutPut.returnErrorPage(page, e);错误页面
// e.printStackTrace();
// pageErrorOutPut.returnErrorPage(page, e); //错误页面
}
}
//结果入库之后再更新lastId.注意，处理一批才会存一次
updateLastId(site,lastId + pages.size());
lastId = getLastId(site);
pages = getPages(site,lastId);

2
new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java
View File

@ -95,7 +95,7 @@ class ReExtractThread implements Runnable{
saveResult(site, rawPage);
}catch (Exception e){
e.printStackTrace();
// pageErrorOutPut.returnErrorPage(page, e);错误页面
// pageErrorOutPut.returnErrorPage(page, e); //错误页面
}
}
//更新抽取游标

13
new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java
View File

@ -12,7 +12,7 @@ import java.util.List;
* Description：抽取openhub中多链接的项目
*/
public class ExtractMutilLink4Openhub implements PageProcessor{
private Site site = Site.me().setRetryTimes(10).setSleepTime(500);
private Site site = Site.me().setRetryTimes(5).setSleepTime(500);
private static List<String> homepages;
private static Page _page;
@ -40,27 +40,28 @@ public class ExtractMutilLink4Openhub implements PageProcessor{
}
}catch(Exception e) {
}
return ExtractMutilLink4Openhub.homepages;
}
/**
* 获取多个link，用“;”间隔
* 获取多个link，用“;”间隔.此方法为二次抽取的入口方法
* @param url
* @return
*/
public String extractLinks(String url){
String homepage = "";
doExtract(url);
doExtract(url); //（如果成功了的话）得到了homepages
try {
List<String> tempPages = getHomepages(url);
List<String> tempPages = getHomepages(url); //以防初次抽取的homepages没抽成功，若没成功的话通过getHomepages继续抽
if (tempPages != null && !tempPages.isEmpty()) {
for (String link : tempPages) {
if(!homepage.contains(link)){//避免加入重复的链接
homepage += link + ";";
}
}
homepage = homepage.substring(0, homepage.lastIndexOf(";"));
homepage = homepage.substring(0, homepage.lastIndexOf(";")); //将结果最后的“；”去除
}
}catch (Exception e){
e.printStackTrace();
@ -89,7 +90,7 @@ public class ExtractMutilLink4Openhub implements PageProcessor{
* 抽取Homepage核心方法
* @param url 抽取目标url链接
*/
public static void doExtract(String url){
public static void doExtract(String url) {
Spider.create(extractMutilLink4Openhub).addUrl(new String("https://www.openhub.net"+url)).thread(1).run();
}