linkInfos) {
- // this.linkInfos = linkInfos;
- // }
+
+
+ public String getValidHomepage(String homePage){
+ String result = homePage;
+ //目前的Homepage抽取规则，以“/”开头的如"/p/jQuery"均是多链接的情形
+ if(!homePage.equals("") && homePage.startsWith("/")){
+ //获取Homepage列表中的Homepage
+ result = new ExtractMutilLink4Openhub().extractLinks(homePage);
+ }
+ return result;
+ }
}
diff --git a/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java b/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
index b1ebdbedb..e740c11b8 100644
--- a/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
+++ b/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
@@ -20,13 +20,17 @@ import core.Page;
import core.ValidateExtractor;
import us.codecraft.webmagic.model.annotation.ExtractBy;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
@ExtractBy("//body[@id='pg_project']/div[@id='page-body']/article[@id='project']")
// 限定抽取区域
public class sourceforge_Model implements AfterExtractor, ValidateExtractor{
///////////////
-@ExtractBy("//a[@id='homepage']/@href")
+@ExtractBy("//a[@id='homepage']/@href " +
+ " | //*[@class='homepage-link']/a/@href")
private String homepage;
//////////////////
@@ -609,4 +613,20 @@ public void setHomepage(String homepage) {
+ ", pageMD5=" + pageMd5 + ", history=" + history + "]";
}
+ /**
+ * 正则验证homepage的有效性
+ * @param homepage
+ * @return
+ */
+ private String checkHomepage(String homepage){
+ String newHomepage = homepage;
+ String key = "^([hH][tT]{2}[pP]://|[hH][tT]{2}[pP][sS]://)(([a-zA-Z0-9\\._-]+\\.[a-zA-Z]{2,6})|([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}))(:[0-9]{1,4})*(/[a-zA-Z0-9\\&%_\\./-~-]*)?";
+ Pattern pattern = Pattern.compile(key,Pattern.CASE_INSENSITIVE);
+ //Homepage无效则使用页面url
+ if(null == homepage || "".equals(homepage) || !pattern.matcher(newHomepage).matches()){
+ newHomepage = this.getUrl();
+ }
+ return newHomepage;
+ }
+
}
diff --git a/new_osseanextractor/src/main/java/net/trustie/one/Application.java b/new_osseanextractor/src/main/java/net/trustie/one/Application.java
index 95968e5fe..18e5487d6 100644
--- a/new_osseanextractor/src/main/java/net/trustie/one/Application.java
+++ b/new_osseanextractor/src/main/java/net/trustie/one/Application.java
@@ -18,6 +18,7 @@ import net.trustie.dao.RecordDao;
import net.trustie.downloader.DataBasePageErrorOutPut;
import net.trustie.downloader.PageDao;
import net.trustie.model.cto51_blog_Model;
+import net.trustie.utils.Constant;
import net.trustie.utils.DateHandler;
import org.springframework.beans.BeansException;
@@ -30,13 +31,22 @@ import core.ModelPipeline;
import core.PageModelPipeline;
import extension.RawPage;
+/**
+ * Title:抽取服务程序入口
+ * Description:利用抽取模板，从爬虫数据库中抽取站点的项目、文档信息，
+ * 抽取至抽取数据库中对应站点的分表中
+ * @author
+ * @date 2016年12月22日
+ */
@Component("Application")
public class Application {
-
- private String sitesPath = "tasks.txt";
- //indicate if a site is being extracted
+ /** 站点列表配置文件路径 */
+ private String sitesPath = Constant.SITE_PATH;
+ /** 抽取状态：indicate if a site is being extracted ，状态为true时表示正在抽取中无需重复抽取*/
public static Map extractState = new HashMap();
- private ExecutorService pool = Executors.newFixedThreadPool(20);
+ /** 线程池，用以装载抽取线程 参数：20为最大线程数 */
+ private ExecutorService pool = Executors.newFixedThreadPool(Constant.POOL_MAX_SIZE);
+
public static void main(String args[]){
((Application)AppContext.appContext.getBean("Application")).startJob();
}
@@ -44,23 +54,23 @@ public class Application {
public void startJob() {
while(true){
//从文件中读取要抽取哪些站点
- List sites = readSites();
+ List sites = readSitesFromConfig();
if(sites.size() == 0)
continue;
-
- //看site被抽取过没，state为true说明已经抽取过了
+
for( String site : sites){
Boolean state = extractState.get(site);
if( state == null || state == false){
extractState.put(site, true);
}
- else if(state == true){
+ else if(state == true){//抽取中，无需重复抽取
continue;
}
//用线程池抽取各个站点
ExtractThread et = (ExtractThread)AppContext.appContext.getBean("extractThread");
et.setSite(site);
- pool.execute(et);
+ //提交任务到线程池，无返回值
+ pool.execute(et);
}
try {
System.out.println(Thread.currentThread().getName() + ": " +
@@ -76,7 +86,11 @@ public class Application {
}
- private List readSites() {
+ /**
+ * 读取站点配置文件
+ * @return 站点列表
+ */
+ private List readSitesFromConfig() {
File file = new File(sitesPath);
LinkedList tasks = new LinkedList();
BufferedReader reader = null;
@@ -101,155 +115,5 @@ public class Application {
}
}
-@Component("extractThread")
-@Scope("prototype")
-class ExtractThread implements Runnable{
-
- private String site;
- private static final int BatchExtractSize = 100;
- public List modelName = new ArrayList();
- private Class pageModel;
- @Resource
- private PageDao pagedao;
- @Qualifier("errorPageToDB")
- @Autowired
- private DataBasePageErrorOutPut pageErrorOutPut;
-
- @Resource
- private RecordDao recordDao;
- public ExtractThread(){
-
- }
- public ExtractThread(String site){
- this.site = site;
- }
- public void setSite(String site){
- this.site = site;
- }
- public void run() {
- //设置threadname 以便查看输出日志
- Thread.currentThread().setName(site + "_thread");
- this.pageErrorOutPut.setTableName(site + "_error_page");
- try {
-
- System.out.println(Thread.currentThread().getName() + ": " +
- "extract begain" + " @ " + DateHandler.getExtractTime());
- if(site.equals("51cto_blog"))
- pageModel = cto51_blog_Model.class;
- else
- pageModel = Class.forName("net.trustie.model." + site +"_Model");
- modelName.add(pageModel.getCanonicalName());
-
- //获取该站点的抽取模板和要抽取的html页面
- int lastId = getLastId(site);
- List pages = getPages(site,lastId);
-
- Extractor extractor = new Extractor();
- RawPage result = null;
- while(pages.size() > 0){
-
- for( RawPage page : pages){
- try{
- result = extractor.extract(page,pageModel);
- //持久化 并 更新抽取历史
- saveResult(site,result);
-
-
- }catch (Exception e){
- e.printStackTrace();
- pageErrorOutPut.returnErrorPage(page, e);
- }
- }
- updateLastId(site,lastId + pages.size());
- lastId = getLastId(site);
- pages = getPages(site,lastId);
- }
- Application.extractState.put(site, false);
- System.out.println(Thread.currentThread().getName() + ": " +
- "extract over, id=" + lastId + " @ " + DateHandler.getExtractTime());
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
- } catch(Exception e){
- System.out.println(e.getMessage());
- e.printStackTrace();
- System.out.println(Thread.currentThread().getName() + ": " +" 发生异常 ");
- }
-
- }
- private void saveResult(String site, RawPage rawPage ) {
-
- ModelPipeline pl = new ModelPipeline();
- try {
- pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
- } catch (BeansException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- if (!rawPage.getPage().getResultItems().isSkip()) {
- pl.process(rawPage.getPage().getResultItems(), null);
- }
- if (!rawPage.getPage()
- .isAllResultSkip(
- modelName.toArray(new String[modelName
- .size()]))) {
- rawPage.setStored(true);
- }else{
-
- }
-
- if (!rawPage.isExtracted() || !rawPage.isStored())
- pageErrorOutPut.returnErrorPage(rawPage,
- "May Caused By Model Problem! Or Page Error!");
-
-
- }
- private void updateLastId(String site, int currentId) {
- recordDao.updateRecord(site,currentId);
- }
-
- private void saveResult(String site, List results) {
- //System.out.println("save result");
- ModelPipeline pl = new ModelPipeline();
- try {
- pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
- } catch (BeansException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- for(RawPage rawPage : results){
- if (!rawPage.getPage().getResultItems().isSkip()) {
- pl.process(rawPage.getPage().getResultItems(), null);
- }
- if (!rawPage.getPage()
- .isAllResultSkip(
- modelName.toArray(new String[modelName
- .size()]))) {
- rawPage.setStored(true);
- }else{
- System.out.println(rawPage.getPage().getResultItems()+"*****************************");
- }
-
- if (!rawPage.isExtracted() || !rawPage.isStored())
- pageErrorOutPut.returnErrorPage(rawPage,
- "May Caused By Model Problem! Or Page Error!");
- }
- }
-
- private List getPages(String site, int lastId) {
- LinkedList pages = new LinkedList();
- pages = pagedao.getDetailPages(site + "_html_detail",lastId,BatchExtractSize);
- return pages;
- }
-
- private int getLastId(String site) {
- Integer lastRecord = recordDao.getLastRecord(site);
- if(lastRecord == null){
- lastRecord = pagedao.getMinId(site + "_html_detail") - 1;
- recordDao.insertRecord(site,lastRecord);
- }
- return lastRecord;
- }
-}
diff --git a/new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java b/new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java
new file mode 100644
index 000000000..e85e8a24e
--- /dev/null
+++ b/new_osseanextractor/src/main/java/net/trustie/one/ExtractThread.java
@@ -0,0 +1,206 @@
+package net.trustie.one;
+
+import core.ModelPipeline;
+import core.PageModelPipeline;
+import extension.RawPage;
+import net.trustie.dao.RecordDao;
+import net.trustie.downloader.DataBasePageErrorOutPut;
+import net.trustie.downloader.PageDao;
+import net.trustie.model.cto51_blog_Model;
+import net.trustie.utils.Constant;
+import net.trustie.utils.DateHandler;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Title:抽取线程
+ * Description:抽取项目的任务主线程
+ * @author liuyizhou
+ * @date 2016年12月22日
+ */
+@Component("extractThread")
+@Scope("prototype")
+public class ExtractThread implements Runnable{
+ /** 待抽取的站点名称 */
+ private String site;
+ /** 待抽取结果条数限制 */
+ private static final int BatchExtractSize = Constant.BATCH_EXTRACT_SIZE;
+ /** 模板名称列表 */
+ public List modelName = new ArrayList();
+ /** 动态生成站点模板 */
+ private Class pageModel;
+ @Resource
+ private PageDao pagedao;
+
+ /** 输出错误页面 */
+ @Qualifier("errorPageToDB")
+ @Autowired
+ private DataBasePageErrorOutPut pageErrorOutPut;
+
+ /** records表操作接口实例化 */
+ @Resource
+ private RecordDao recordDao;
+
+ public ExtractThread(){
+
+ }
+
+ public ExtractThread(String site){
+ setSite(site);
+ }
+
+ public void setSite(String site){
+ this.site = site;
+ }
+
+ public void run() {
+ //设置thread name 以便查看输出日志
+ Thread.currentThread().setName(site + "_thread");
+ this.pageErrorOutPut.setTableName(site + "_error_page");
+ try {
+
+ System.out.println(Thread.currentThread().getName() + ": " +
+ "extract begain" + " @ " + DateHandler.getExtractTime());
+
+ if(site.equals("51cto_blog"))
+ pageModel = cto51_blog_Model.class;
+ else
+ pageModel = Class.forName("net.trustie.model." + site +"_Model");
+
+ modelName.add(pageModel.getCanonicalName());
+
+ //获取该站点的抽取模板和要抽取的html页面
+ int lastId = getLastId(site);
+ List pages = getPages(site,lastId);
+
+ Extractor extractor = new Extractor();
+ RawPage result = null;
+ while(pages.size() > 0){
+
+ for( RawPage page : pages){
+ try{
+ result = extractor.extract(page,pageModel);
+ //持久化 并 更新抽取历史
+ saveResult(site,result);
+
+
+ }catch (Exception e){
+ e.printStackTrace();
+ pageErrorOutPut.returnErrorPage(page, e);
+ }
+ }
+ updateLastId(site,lastId + pages.size());
+ lastId = getLastId(site);
+ pages = getPages(site,lastId);
+ }
+ Application.extractState.put(site, false);
+ System.out.println(Thread.currentThread().getName() + ": " +
+ "extract over, id=" + lastId + " @ " + DateHandler.getExtractTime());
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ } catch(Exception e){
+ System.out.println(e.getMessage());
+ e.printStackTrace();
+ System.out.println(Thread.currentThread().getName() + ": " +" 发生异常 ");
+ }
+
+
+ }
+ private void saveResult(String site, RawPage rawPage ) {
+
+ ModelPipeline pl = new ModelPipeline();
+ try {
+ pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
+ } catch (BeansException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ if (!rawPage.getPage().getResultItems().isSkip()) {
+ pl.process(rawPage.getPage().getResultItems(), null);
+ }
+ if (!rawPage.getPage()
+ .isAllResultSkip(
+ modelName.toArray(new String[modelName
+ .size()]))) {
+ rawPage.setStored(true);
+ }else{
+
+ }
+
+ if (!rawPage.isExtracted() || !rawPage.isStored())
+ pageErrorOutPut.returnErrorPage(rawPage,
+ "May Caused By Model Problem! Or Page Error!");
+
+
+ }
+ private void updateLastId(String site, int currentId) {
+ recordDao.updateRecord(site,currentId);
+ }
+
+ private void saveResult(String site, List results) {
+ //System.out.println("save result");
+ ModelPipeline pl = new ModelPipeline();
+ try {
+ pl.put(pageModel,(PageModelPipeline) AppContext.appContext.getBean(site + "_pipeline"));
+ } catch (BeansException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ for(RawPage rawPage : results){
+ if (!rawPage.getPage().getResultItems().isSkip()) {
+ pl.process(rawPage.getPage().getResultItems(), null);
+ }
+ if (!rawPage.getPage()
+ .isAllResultSkip(
+ modelName.toArray(new String[modelName
+ .size()]))) {
+ rawPage.setStored(true);
+ }else{
+ System.out.println(rawPage.getPage().getResultItems()+"*****************************");
+ }
+
+ if (!rawPage.isExtracted() || !rawPage.isStored())
+ pageErrorOutPut.returnErrorPage(rawPage,
+ "May Caused By Model Problem! Or Page Error!");
+ }
+ }
+
+ /**
+ * 获取站点site_html_detail表中，id大于lastId的 BatchExtractSize 条记录
+ * @param site
+ * @param lastId
+ * @return
+ */
+ private List getPages(String site, int lastId) {
+ LinkedList pages = new LinkedList();
+ pages = pagedao.getDetailPages(site + "_html_detail",lastId,BatchExtractSize);
+ return pages;
+ }
+
+ /**
+ * 获取站点site的最后id
+ * @param site
+ * @return
+ */
+ private int getLastId(String site) {
+ /** 从records表中，获取最新的抽取记录的id */
+ Integer lastRecord = recordDao.getLastRecord(site);
+ if(lastRecord == null){//最新抽取记录为空，则新增
+ //从对应站点HTML详情表site_html_detail中，获取最新的id
+ lastRecord = pagedao.getMinId(site + "_html_detail") - 1;//TODO 为什么是min(ID)
+ //最新id插入records表
+ recordDao.insertRecord(site,lastRecord);
+ }
+ return lastRecord;
+ }
+}
+
diff --git a/new_osseanextractor/src/main/java/net/trustie/one/Extractor.java b/new_osseanextractor/src/main/java/net/trustie/one/Extractor.java
index 044686907..b1c7c8cde 100644
--- a/new_osseanextractor/src/main/java/net/trustie/one/Extractor.java
+++ b/new_osseanextractor/src/main/java/net/trustie/one/Extractor.java
@@ -10,6 +10,10 @@ import core.ModelPageProcessor;
import core.Pipeline;
import extension.RawPage;
+/**
+ * 抽取核心类
+ * 主要执行
+ */
public class Extractor {
public List extract(List pages, Class modelClass ) {
diff --git a/new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java b/new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java
new file mode 100644
index 000000000..a88c412fe
--- /dev/null
+++ b/new_osseanextractor/src/main/java/net/trustie/one/OpenhubReExtractor.java
@@ -0,0 +1,163 @@
+package net.trustie.one;
+
+import core.ModelPipeline;
+import core.ModelPipeline4Update;
+import core.PageModelPipeline;
+import core.PageModelPipeline4Update;
+import extension.RawPage;
+import net.trustie.dao.RecordDao;
+import net.trustie.downloader.DataBasePageErrorOutPut;
+import net.trustie.downloader.PageDao;
+import net.trustie.utils.Constant;
+import net.trustie.utils.DateHandler;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Created by LouAnt on 2017/2/13 0013
+ * Description：Openhub 社区项目Homepage字段重新抽取
+ */
+@Component("ReExtractor")
+public class OpenhubReExtractor {
+ /** 抽取状态：indicate if a site is being extracted ，状态为true时表示正在抽取中无需重复抽取*/
+ public static Map extractState = new HashMap();
+ /** 线程池，用以装载抽取线程 参数：20为最大线程数 */
+ private ExecutorService pool = Executors.newFixedThreadPool(Constant.POOL_MAX_SIZE);
+
+ public static void main(String[] args){
+ ((OpenhubReExtractor) AppContext.appContext.getBean("ReExtractor")).start();
+ }
+
+ public void start(){
+ while(true){
+ ReExtractThread reExtractThread = (ReExtractThread)AppContext.appContext.getBean("reExtractThread");
+ pool.execute(reExtractThread);
+ }
+ }
+}
+
+/**
+ * 重新抽取的主线程
+ */
+@Component("reExtractThread")
+@Scope("prototype")
+class ReExtractThread implements Runnable{
+ /** 待抽取的站点名称 */
+ private String site;
+ /** 待抽取结果条数限制 */
+ private static final int BatchExtractSize = Constant.BATCH_EXTRACT_SIZE;
+ /** 模板名称列表 */
+ public List modelName = new ArrayList();
+ /** 动态生成站点模板 */
+ private Class pageModel;
+ @Resource
+ private PageDao pagedao;
+
+ /** 输出错误页面 */
+ @Qualifier("errorPageToDB")
+ @Autowired
+ private DataBasePageErrorOutPut pageErrorOutPut;
+
+ /** records表操作接口实例化 */
+ @Resource
+ private RecordDao recordDao;
+
+ public void run(){
+ /**站点名称*/
+ String site = "openhub";
+ Thread.currentThread().setName(site + "reextractor_thread");
+ this.pageErrorOutPut.setTableName(site+"_error_page");
+
+ try {
+ pageModel = Class.forName("net.trustie.model."+site+"_Model");
+
+ modelName.add(pageModel.getCanonicalName());
+
+ int lastId = getLastId(site);
+
+ List pages = getPages(site,lastId);
+
+ Extractor extractor = new Extractor();
+ RawPage rawPage = null;
+
+ while(pages.size() > 0){
+ for(RawPage page : pages){
+ try {
+ rawPage = extractor.extract(page, pageModel);
+ saveResult(site, rawPage);
+ }catch (Exception e){
+ e.printStackTrace();
+ // pageErrorOutPut.returnErrorPage(page, e);错误页面
+ }
+ }
+ //更新抽取游标
+ updateLastId(site,lastId + pages.size());
+ lastId = getLastId(site);
+ pages = getPages(site,lastId);
+ }
+ Application.extractState.put(site, false);
+ System.out.println(Thread.currentThread().getName() + ": " +
+ "extract over, id=" + lastId + " @ " + DateHandler.getExtractTime());
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ } catch(Exception e){
+ System.out.println(e.getMessage());
+ e.printStackTrace();
+ System.out.println(Thread.currentThread().getName() + ": " +" 发生异常 ");
+ }
+ }
+
+ private void saveResult(String site,RawPage rawPage){
+ ModelPipeline4Update pl = new ModelPipeline4Update();
+ try {
+ pl.put(pageModel, (PageModelPipeline4Update) AppContext.appContext.getBean(site + "_pipeline_4update"));
+ }catch (BeansException e){
+ e.printStackTrace();
+ }
+
+ if(!rawPage.getPage().getResultItems().isSkip()){
+ pl.processUpdate(rawPage.getPage().getResultItems(),rawPage.getUrl(),null);
+ }
+ }
+
+ /**
+ * 获取站点site_html_detail表中，id大于lastId的 BatchExtractSize 条记录
+ * @param site
+ * @param lastId
+ * @return
+ */
+ private List getPages(String site, int lastId) {
+ LinkedList pages = new LinkedList();
+ pages = pagedao.getDetailPages(site + "_html_detail",lastId,BatchExtractSize);
+ return pages;
+ }
+
+ /**
+ * 获取站点site的最后id
+ * @param site
+ * @return
+ */
+ private int getLastId(String site) {
+ /** 从records表中，获取最新的抽取记录的id */
+ Integer lastRecord = recordDao.getLastRecord(site+"_reextractor");
+ if(lastRecord == null){//最新抽取记录为空，则新增
+ //从对应站点HTML详情表site_html_detail中，获取最新的id
+ lastRecord = pagedao.getMinId(site + "_html_detail") - 1;
+ //最新id插入records表
+ recordDao.insertRecord(site+"_reextractor",lastRecord);
+ }
+ return lastRecord;
+ }
+
+ private void updateLastId(String site, int currentId) {
+ recordDao.updateRecord(site+"_reextractor",currentId);
+ }
+}
diff --git a/new_osseanextractor/src/main/java/net/trustie/pipeline/OpenHubProject_pipeline_4update.java b/new_osseanextractor/src/main/java/net/trustie/pipeline/OpenHubProject_pipeline_4update.java
new file mode 100644
index 000000000..3c030bf8c
--- /dev/null
+++ b/new_osseanextractor/src/main/java/net/trustie/pipeline/OpenHubProject_pipeline_4update.java
@@ -0,0 +1,34 @@
+package net.trustie.pipeline;
+
+import core.PageModelPipeline4Update;
+import net.trustie.dao.OpenHubProject4Update_Dao;
+import net.trustie.model.openhub_Model;
+import us.codecraft.webmagic.Task;
+
+import javax.annotation.Resource;
+
+import org.springframework.stereotype.Component;
+
+import net.trustie.dao.OpenHubProject_Dao;
+import net.trustie.model.openhub_Model;
+import core.PageModelPipeline;
+
+/**
+ * Created by LouAnt on 2017/2/14 0014
+ * Description：
+ */
+@Component("openhub_pipeline_4update")
+public class OpenHubProject_pipeline_4update implements PageModelPipeline4Update {
+
+ @Resource
+ private OpenHubProject4Update_Dao openHubProject4Update_dao;
+ /**
+ * 更新Homepage专用
+ * @param openhub_model
+ * @param task
+ */
+ @Override
+ public void processUpdate(openhub_Model openhub_model, String url, Task task){
+ openHubProject4Update_dao.updateHomepage(openhub_model.getHomepage(),url);
+ }
+}
diff --git a/new_osseanextractor/src/main/java/net/trustie/utils/Constant.java b/new_osseanextractor/src/main/java/net/trustie/utils/Constant.java
new file mode 100644
index 000000000..474b06bb7
--- /dev/null
+++ b/new_osseanextractor/src/main/java/net/trustie/utils/Constant.java
@@ -0,0 +1,18 @@
+package net.trustie.utils;
+
+/**
+ * Title:公共变量类
+ * Description:用于存放全局静态变量
+ * @author liuyizhou
+ * @date 2016年12月22日
+ */
+public class Constant {
+ /** 程序根目录*/
+ private final static String CURRENT_PATH = System.getProperty("user.dir");
+ /** 开源站点列表配置文件路径 */
+ public final static String SITE_PATH = CURRENT_PATH + "\\tasks.txt";
+ /** 线程池最大线程数 */
+ public final static int POOL_MAX_SIZE = 20;
+ /** 抽取站点结果时，待抽取结果条数限制*/
+ public final static int BATCH_EXTRACT_SIZE = 100;
+}
diff --git a/new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java b/new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java
new file mode 100644
index 000000000..b4002d096
--- /dev/null
+++ b/new_osseanextractor/src/main/java/net/trustie/utils/ExtractMutilLink4Openhub.java
@@ -0,0 +1,100 @@
+package net.trustie.utils;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * Created by LouAnt on 2017/2/10 0010
+ * Description：抽取openhub中多链接的项目
+ */
+public class ExtractMutilLink4Openhub implements PageProcessor{
+ private Site site = Site.me().setRetryTimes(5).setSleepTime(500);
+
+ private static List homepages;
+ private static Page _page;
+
+ static ExtractMutilLink4Openhub extractMutilLink4Openhub = new ExtractMutilLink4Openhub();
+
+ @Override
+ public void process(Page page) {
+ //抽取Homepage列表，Html格式参考 https://www.openhub.net/p/zimbra/links
+ ExtractMutilLink4Openhub.homepages = page.getHtml().xpath("//div[@class='links center-block']/div[1]/div[2]/div/h5/a").links().all();
+ }
+
+ @Override
+ public Site getSite() {
+ return site;
+ }
+
+ public List getHomepages(String url){
+ try{
+ int count = 5;
+ //为防止一次抽取不成功，抽取5次
+ while(ExtractMutilLink4Openhub.homepages==null && count > 0 ) {
+ ExtractMutilLink4Openhub.doExtract(url);
+ count--;
+ }
+ }catch(Exception e) {
+
+ }
+ return ExtractMutilLink4Openhub.homepages;
+ }
+
+ /**
+ * 获取多个link，用“;”间隔
+ * @param url
+ * @return
+ */
+ public String extractLinks(String url){
+ String homepage = "";
+ doExtract(url);
+ try {
+ List tempPages = getHomepages(url);
+ if (tempPages != null && !tempPages.isEmpty()) {
+ for (String link : tempPages) {
+ if(!homepage.contains(link)){//避免加入重复的链接
+ homepage += link + ";";
+ }
+ }
+ homepage = homepage.substring(0, homepage.lastIndexOf(";"));
+ }
+ }catch (Exception e){
+ e.printStackTrace();
+ }
+ return homepage;
+ }
+
+ /**
+ * 获取第一个link作为Homepage
+ * @param url
+ * @return
+ */
+ public String extractLink(String url){
+ String homepage = url;
+ doExtract(url);
+ try {
+ List tempPages = getHomepages(url);
+ homepage = (tempPages != null && !tempPages.isEmpty()) ? tempPages.get(0) : url;
+ }catch (Exception e){
+ e.printStackTrace();
+ }
+ return homepage;
+ }
+
+ /**
+ * 抽取Homepage核心方法
+ * @param url 抽取目标url链接
+ */
+ public static void doExtract(String url){
+ Spider.create(extractMutilLink4Openhub).addUrl(new String("https://www.openhub.net"+url)).thread(1).run();
+ }
+
+ public static void main(String[] args){
+ String result = new ExtractMutilLink4Openhub().extractLink("/p/jQuery/links");
+ System.out.println(result);
+ }
+}
diff --git a/project_match/bin/start_get_synonyms.sh~HEAD b/project_match/bin/start_get_synonyms.sh~HEAD
new file mode 100644
index 000000000..4001d35ff
--- /dev/null
+++ b/project_match/bin/start_get_synonyms.sh~HEAD
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+
+#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
+
+tmp='./target/classes':$tmp
+tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+tmp='./bin/resources':$tmp
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+java -classpath $CLASSPATH com.ossean.GetSynonyms >>log/getsynonyms_2017.log 2>&1 &
diff --git a/project_match/bin/start_get_synonyms.sh~liuyizhou b/project_match/bin/start_get_synonyms.sh~liuyizhou
new file mode 100644
index 000000000..c8e21dce8
--- /dev/null
+++ b/project_match/bin/start_get_synonyms.sh~liuyizhou
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+find ./target/classes -name "*.properties"|xargs rm -f
+find ./target/classes -name "*.xml"|xargs rm -f
+find ./target/classes -name "*.dic"|xargs rm -f
+
+#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
+
+tmp='./target/classes':$tmp
+tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
+tmp='./bin/resources':$tmp
+CLASSPATH=$tmp:$CLASSPATH
+
+
+echo $CLASSPATH
+JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
+#echo JVM_ARGS=$JVM_ARGS
+#ulimit -n 400000
+#echo "" > nohup.out
+java $JVM_ARGS -classpath $CLASSPATH com.ossean.GetSynonyms >>log/getsynonyms_2017.log 2>&1 &
\ No newline at end of file