diff --git a/new_osseanextractor/resources/spring/applicationContext-myBatis.xml b/new_osseanextractor/resources/spring/applicationContext-myBatis.xml
index f6e153608..fd83081b3 100644
--- a/new_osseanextractor/resources/spring/applicationContext-myBatis.xml
+++ b/new_osseanextractor/resources/spring/applicationContext-myBatis.xml
@@ -10,7 +10,7 @@
-
+
@@ -27,7 +27,7 @@
-
+
diff --git a/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java b/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java
index be392d4cf..e5ac1ca69 100644
--- a/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java
+++ b/new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java
@@ -17,16 +17,16 @@ import core.ValidateExtractor;
@ExtractBy("//div[@id='OSC_Content']/div[@class='ProjectPage']/div[@class='ProjectMain']/div[@class='Project']")
public class oschina_project_Model implements AfterExtractor, ValidateExtractor {
-//////
-@ExtractBy("//*[@id='Body']/ul[2]/li[1]/a/@href")
-private String homepage;
+ // ////
+ @ExtractBy("//*[@id='Body']/ul[2]/li[1]/a/@href")
+ private String homepage;
-@ExtractBy("//*[@id='Body']/ul[2]/li[2]/a/@href")
-private String doc_url;
+ @ExtractBy("//*[@id='Body']/ul[2]/li[2]/a/@href")
+ private String doc_url;
-@ExtractBy("//*[@id='Body']/ul[2]/li[3]/a/@href")
-private String download_url;
-/////////
+ @ExtractBy("//*[@id='Body']/ul[2]/li[3]/a/@href")
+ private String download_url;
+ // ///////
private String projectUrl;
@@ -69,34 +69,37 @@ private String download_url;
private List attrs;
private String url_md5;
- private String source="oschina";
-
+ private String source = "oschina";
+
private String tag;
-
+
private List tags;
+
@Override
public void afterProcess(Page page) {
+
+ //System.out.println("prjcategory: " + projectCategory);
List tags = new LinkedList();
- if(projectCategory.contains("/")){
- String[] tags_1 =projectCategory.split("/");
- for(String item : tags_1){
+ if (projectCategory.contains("/")) {
+ String[] tags_1 = projectCategory.split("/");
+ for (String item : tags_1) {
tags.add(item);
}
- }else{
+ } else {
tags.add(projectCategory);
}
-
+
this.projectCategory = StringHandler.combineTags(tags);
- //System.out.println(StringHandler.combineTags(tags));
-
- //处理homepage, doc_url, down_url等
-
- this.homepage="http://www.oschina.net"+homepage;
- this.doc_url="http://www.oschina.net"+doc_url;
- this.download_url="http://www.oschina.net"+download_url;
+ // System.out.println(StringHandler.combineTags(tags));
+
+ // 处理homepage, doc_url, down_url等
+
+ this.homepage = "http://www.oschina.net" + homepage;
+ this.doc_url = "http://www.oschina.net" + doc_url;
+ this.download_url = "http://www.oschina.net" + download_url;
// 处理projectUrl
- this.projectUrl = page.getPageUrl();
+ this.projectUrl = page.getPageUrl();
this.url_md5 = DigestUtils.md5Hex(this.projectUrl);
// 处理projectShortName
if (StringUtils.isNotBlank(this.projectShortName))
@@ -144,25 +147,24 @@ private String download_url;
StringHandler.extractHtml(s, "//allText()"),
"\\d+年\\d+月\\d+日");
- this.IncludedTime = DateHandler
- .formatAllTypeDate(this.IncludedTime,page.getTime());
+ this.IncludedTime = DateHandler.formatAllTypeDate(
+ this.IncludedTime, page.getTime());
}
}
+
}
@Override
public void validate(Page page) {
+
+
-// System.out.println("title: " + this.projectTitle);
-// System.out.println("shortName: " + this.projectShortName);
-// System.out.println("usedNum/hoseNum: " + this.usedNum + "," + this.housedNum);
-// System.out.println("includedTime/extractTime: " + this.IncludedTime+ "," +this.exteactTime);
if (StringHandler.isAtLeastOneBlank(this.projectTitle,
this.projectCategory)) {
page.setResultSkip(this, true);
return;
}
-
+
if (StringHandler.isAtLeastOneBlank(this.projectShortName,
this.projectCategory)) {
page.setResultSkip(this, true);
@@ -178,33 +180,33 @@ private String download_url;
page.setResultSkip(this, true);
}
-////////
-
-public String getHomepage() {
-return homepage;
-}
+ // //////
-public void setHomepage(String homepage) {
-this.homepage = homepage;
-}
+ public String getHomepage() {
+ return homepage;
+ }
-public String getDoc_url() {
-return doc_url;
-}
+ public void setHomepage(String homepage) {
+ this.homepage = homepage;
+ }
-public void setDoc_url(String doc_url) {
-this.doc_url = doc_url;
-}
+ public String getDoc_url() {
+ return doc_url;
+ }
-public String getDownload_url() {
-return download_url;
-}
+ public void setDoc_url(String doc_url) {
+ this.doc_url = doc_url;
+ }
-public void setDownload_url(String download_url) {
-this.download_url = download_url;
-}
+ public String getDownload_url() {
+ return download_url;
+ }
-///////////
+ public void setDownload_url(String download_url) {
+ this.download_url = download_url;
+ }
+
+ // /////////
public String getProjectUrl() {
return projectUrl;
diff --git a/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java b/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
index 62a072840..b1ebdbedb 100644
--- a/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
+++ b/new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java
@@ -26,7 +26,7 @@ public class sourceforge_Model implements AfterExtractor, ValidateExtractor{
///////////////
-@ExtractBy("//*[@id='overview']/div[2]/section/p[3]/a/@href")
+@ExtractBy("//a[@id='homepage']/@href")
private String homepage;
//////////////////
@@ -362,8 +362,7 @@ private String homepage;
public void validate(Page page) {
this.name = this.name.trim();
//System.out.println("..." + this.name +"...");
- if (StringHandler.isAtLeastOneBlank(this.name, this.platform,
- this.categories, this.url)) {
+ if (StringHandler.isAtLeastOneBlank(this.name, this.homepage,this.url)) {
page.setResultSkip(this, true);
return;
}
@@ -372,11 +371,12 @@ private String homepage;
page.setResultSkip(this, true);
return;
}
- ;
+
if (!DateHandler.canFormatToDate(this.lastUpdate, this.registeredTime,
this.collectTime)) {
page.setResultSkip(this, true);
}
+
}
diff --git a/new_osseanextractor/src/main/java/net/trustie/one/Application.java b/new_osseanextractor/src/main/java/net/trustie/one/Application.java
index a9b65145c..b2c7578f3 100644
--- a/new_osseanextractor/src/main/java/net/trustie/one/Application.java
+++ b/new_osseanextractor/src/main/java/net/trustie/one/Application.java
@@ -155,7 +155,7 @@ class ExtractThread implements Runnable{
}catch (Exception e){
- System.out.println(e.getMessage());
+ e.printStackTrace();
pageErrorOutPut.returnErrorPage(page, e);
}
}
diff --git a/new_osseanextractor/tasks.txt b/new_osseanextractor/tasks.txt
index beb17c41d..613fd57a0 100644
--- a/new_osseanextractor/tasks.txt
+++ b/new_osseanextractor/tasks.txt
@@ -1,18 +1 @@
-stackoverflow
-oschina_project
-iteye_blog
-csdn_blog
-freecode
-51cto_blog
-apache
-sourceforge
-openhub
-csdn_ask
-neitui
-lagou
-oschina_question
-csdn_topic
-dewen_question
-cnblogs_q_solved
-cnblogs_q_unsolved
-cnblogs_news
\ No newline at end of file
+sourceforge
\ No newline at end of file