From 4ad9dee4c9e2fbbfd4ed8f89978cf016f9dfd517 Mon Sep 17 00:00:00 2001 From: StarLee Date: Sun, 27 Mar 2016 22:49:46 +0800 Subject: [PATCH] modify sourceforge's model --- .../spring/applicationContext-myBatis.xml | 4 +- .../trustie/model/oschina_project_Model.java | 106 +++++++++--------- .../net/trustie/model/sourceforge_Model.java | 8 +- .../java/net/trustie/one/Application.java | 2 +- new_osseanextractor/tasks.txt | 19 +--- 5 files changed, 62 insertions(+), 77 deletions(-) diff --git a/new_osseanextractor/resources/spring/applicationContext-myBatis.xml b/new_osseanextractor/resources/spring/applicationContext-myBatis.xml index f6e153608..fd83081b3 100644 --- a/new_osseanextractor/resources/spring/applicationContext-myBatis.xml +++ b/new_osseanextractor/resources/spring/applicationContext-myBatis.xml @@ -10,7 +10,7 @@ - + @@ -27,7 +27,7 @@ - + diff --git a/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java b/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java index be392d4cf..e5ac1ca69 100644 --- a/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java +++ b/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java @@ -17,16 +17,16 @@ import core.ValidateExtractor; @ExtractBy("//div[@id='OSC_Content']/div[@class='ProjectPage']/div[@class='ProjectMain']/div[@class='Project']") public class oschina_project_Model implements AfterExtractor, ValidateExtractor { -////// -@ExtractBy("//*[@id='Body']/ul[2]/li[1]/a/@href") -private String homepage; + // //// + @ExtractBy("//*[@id='Body']/ul[2]/li[1]/a/@href") + private String homepage; -@ExtractBy("//*[@id='Body']/ul[2]/li[2]/a/@href") -private String doc_url; + @ExtractBy("//*[@id='Body']/ul[2]/li[2]/a/@href") + private String doc_url; -@ExtractBy("//*[@id='Body']/ul[2]/li[3]/a/@href") -private String download_url; -///////// + @ExtractBy("//*[@id='Body']/ul[2]/li[3]/a/@href") + private String download_url; + // /////// private String projectUrl; @@ -69,34 +69,37 @@ private String download_url; private List attrs; private String url_md5; - private String source="oschina"; - + private String source = "oschina"; + private String tag; - + private List tags; + @Override public void afterProcess(Page page) { + + //System.out.println("prjcategory: " + projectCategory); List tags = new LinkedList(); - if(projectCategory.contains("/")){ - String[] tags_1 =projectCategory.split("/"); - for(String item : tags_1){ + if (projectCategory.contains("/")) { + String[] tags_1 = projectCategory.split("/"); + for (String item : tags_1) { tags.add(item); } - }else{ + } else { tags.add(projectCategory); } - + this.projectCategory = StringHandler.combineTags(tags); - //System.out.println(StringHandler.combineTags(tags)); - - //处理homepage， doc_url, down_url等 - - this.homepage="http://www.oschina.net"+homepage; - this.doc_url="http://www.oschina.net"+doc_url; - this.download_url="http://www.oschina.net"+download_url; + // System.out.println(StringHandler.combineTags(tags)); + + // 处理homepage， doc_url, down_url等 + + this.homepage = "http://www.oschina.net" + homepage; + this.doc_url = "http://www.oschina.net" + doc_url; + this.download_url = "http://www.oschina.net" + download_url; // 处理projectUrl - this.projectUrl = page.getPageUrl(); + this.projectUrl = page.getPageUrl(); this.url_md5 = DigestUtils.md5Hex(this.projectUrl); // 处理projectShortName if (StringUtils.isNotBlank(this.projectShortName)) @@ -144,25 +147,24 @@ private String download_url; StringHandler.extractHtml(s, "//allText()"), "\\d+年\\d+月\\d+日"); - this.IncludedTime = DateHandler - .formatAllTypeDate(this.IncludedTime,page.getTime()); + this.IncludedTime = DateHandler.formatAllTypeDate( + this.IncludedTime, page.getTime()); } } + } @Override public void validate(Page page) { + + -// System.out.println("title: " + this.projectTitle); -// System.out.println("shortName: " + this.projectShortName); -// System.out.println("usedNum/hoseNum: " + this.usedNum + "," + this.housedNum); -// System.out.println("includedTime/extractTime: " + this.IncludedTime+ "," +this.exteactTime); if (StringHandler.isAtLeastOneBlank(this.projectTitle, this.projectCategory)) { page.setResultSkip(this, true); return; } - + if (StringHandler.isAtLeastOneBlank(this.projectShortName, this.projectCategory)) { page.setResultSkip(this, true); @@ -178,33 +180,33 @@ private String download_url; page.setResultSkip(this, true); } -//////// - -public String getHomepage() { -return homepage; -} + // ////// -public void setHomepage(String homepage) { -this.homepage = homepage; -} + public String getHomepage() { + return homepage; + } -public String getDoc_url() { -return doc_url; -} + public void setHomepage(String homepage) { + this.homepage = homepage; + } -public void setDoc_url(String doc_url) { -this.doc_url = doc_url; -} + public String getDoc_url() { + return doc_url; + } -public String getDownload_url() { -return download_url; -} + public void setDoc_url(String doc_url) { + this.doc_url = doc_url; + } -public void setDownload_url(String download_url) { -this.download_url = download_url; -} + public String getDownload_url() { + return download_url; + } -/////////// + public void setDownload_url(String download_url) { + this.download_url = download_url; + } + + // ///////// public String getProjectUrl() { return projectUrl; diff --git a/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java b/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java index 62a072840..b1ebdbedb 100644 --- a/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java +++ b/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java @@ -26,7 +26,7 @@ public class sourceforge_Model implements AfterExtractor, ValidateExtractor{ /////////////// -@ExtractBy("//*[@id='overview']/div[2]/section/p[3]/a/@href") +@ExtractBy("//a[@id='homepage']/@href") private String homepage; ////////////////// @@ -362,8 +362,7 @@ private String homepage; public void validate(Page page) { this.name = this.name.trim(); //System.out.println("..." + this.name +"..."); - if (StringHandler.isAtLeastOneBlank(this.name, this.platform, - this.categories, this.url)) { + if (StringHandler.isAtLeastOneBlank(this.name, this.homepage,this.url)) { page.setResultSkip(this, true); return; } @@ -372,11 +371,12 @@ private String homepage; page.setResultSkip(this, true); return; } - ; + if (!DateHandler.canFormatToDate(this.lastUpdate, this.registeredTime, this.collectTime)) { page.setResultSkip(this, true); } + } diff --git a/new_osseanextractor/src/main/java/net/trustie/one/Application.java b/new_osseanextractor/src/main/java/net/trustie/one/Application.java index a9b65145c..b2c7578f3 100644 --- a/new_osseanextractor/src/main/java/net/trustie/one/Application.java +++ b/new_osseanextractor/src/main/java/net/trustie/one/Application.java @@ -155,7 +155,7 @@ class ExtractThread implements Runnable{ }catch (Exception e){ - System.out.println(e.getMessage()); + e.printStackTrace(); pageErrorOutPut.returnErrorPage(page, e); } } diff --git a/new_osseanextractor/tasks.txt b/new_osseanextractor/tasks.txt index beb17c41d..613fd57a0 100644 --- a/new_osseanextractor/tasks.txt +++ b/new_osseanextractor/tasks.txt @@ -1,18 +1 @@ -stackoverflow -oschina_project -iteye_blog -csdn_blog -freecode -51cto_blog -apache -sourceforge -openhub -csdn_ask -neitui -lagou -oschina_question -csdn_topic -dewen_question -cnblogs_q_solved -cnblogs_q_unsolved -cnblogs_news \ No newline at end of file +sourceforge \ No newline at end of file