modify sourceforge's model

new_osseanextractor/resources/spring/applicationContext-myBatis.xml

@@ -10,7 +10,7 @@
 				<property name="url"
 			value="jdbc:mysql://localhost:3306/html_crawler_test?characterEncoding=UTF-8" />
 		<property name="username" value="starlee" />
-		<property name="password" value="starlee" />
+		<property name="password" value="1234" />
 	</bean>
 	<bean id="sqlSessionFactoryOne" class="org.mybatis.spring.SqlSessionFactoryBean">
 		<property name="dataSource" ref="dataSourceOne" />
@@ -27,7 +27,7 @@
 		<property name="url"
 			value="jdbc:mysql://localhost:3306/extract_result?characterEncoding=UTF-8" />
 		<property name="username" value="starlee" />
-		<property name="password" value="starlee"/>
+		<property name="password" value="1234"/>
 	</bean>
 	<bean id="sqlSessionFactoryTwo" class="org.mybatis.spring.SqlSessionFactoryBean">
 		<property name="dataSource" ref="dataSourceTwo" />

new_osseanextractor/src/main/java/net/trustie/model/oschina_project_Model.java

@@ -17,16 +17,16 @@ import core.ValidateExtractor;
 @ExtractBy("//div[@id='OSC_Content']/div[@class='ProjectPage']/div[@class='ProjectMain']/div[@class='Project']")
 public class oschina_project_Model implements AfterExtractor, ValidateExtractor {
 
-//////
-@ExtractBy("//*[@id='Body']/ul[2]/li[1]/a/@href")	
-private String homepage;
+	// ////
+	@ExtractBy("//*[@id='Body']/ul[2]/li[1]/a/@href")
+	private String homepage;
 
-@ExtractBy("//*[@id='Body']/ul[2]/li[2]/a/@href")
-private String doc_url;
+	@ExtractBy("//*[@id='Body']/ul[2]/li[2]/a/@href")
+	private String doc_url;
 
-@ExtractBy("//*[@id='Body']/ul[2]/li[3]/a/@href")
-private String download_url;
-/////////
+	@ExtractBy("//*[@id='Body']/ul[2]/li[3]/a/@href")
+	private String download_url;
+	// ///////
 
 	private String projectUrl;
 
@@ -69,34 +69,37 @@ private String download_url;
 	private List<String> attrs;
 
 	private String url_md5;
-	private String source="oschina";
-	
+	private String source = "oschina";
+
 	private String tag;
-	
+
 	private List<String> tags;
+
 	@Override
 	public void afterProcess(Page page) {
+
 		
+		//System.out.println("prjcategory: " + projectCategory);
 		List<String> tags = new LinkedList<String>();
-		if(projectCategory.contains("/")){
-			String[] tags_1 =projectCategory.split("/");
-			for(String item : tags_1){
+		if (projectCategory.contains("/")) {
+			String[] tags_1 = projectCategory.split("/");
+			for (String item : tags_1) {
 				tags.add(item);
 			}
-		}else{
+		} else {
 			tags.add(projectCategory);
 		}
-		
+
 		this.projectCategory = StringHandler.combineTags(tags);
-		//System.out.println(StringHandler.combineTags(tags));
-		
-		//处理homepage， doc_url, down_url等
-		
-		this.homepage="http://www.oschina.net"+homepage;
-		this.doc_url="http://www.oschina.net"+doc_url;
-		this.download_url="http://www.oschina.net"+download_url;
+		// System.out.println(StringHandler.combineTags(tags));
+
+		// 处理homepage， doc_url, down_url等
+
+		this.homepage = "http://www.oschina.net" + homepage;
+		this.doc_url = "http://www.oschina.net" + doc_url;
+		this.download_url = "http://www.oschina.net" + download_url;
 		// 处理projectUrl
-		this.projectUrl = page.getPageUrl();		
+		this.projectUrl = page.getPageUrl();
 		this.url_md5 = DigestUtils.md5Hex(this.projectUrl);
 		// 处理projectShortName
 		if (StringUtils.isNotBlank(this.projectShortName))
@@ -144,25 +147,24 @@ private String download_url;
 						StringHandler.extractHtml(s, "//allText()"),
 						"\\d+年\\d+月\\d+日");
 
-				this.IncludedTime = DateHandler
-						.formatAllTypeDate(this.IncludedTime,page.getTime());
+				this.IncludedTime = DateHandler.formatAllTypeDate(
+						this.IncludedTime, page.getTime());
 			}
 		}
+		
 	}
 
 	@Override
 	public void validate(Page page) {
+
+		
 		
-//		System.out.println("title: " + this.projectTitle);
-//		System.out.println("shortName: " + this.projectShortName);
-//		System.out.println("usedNum/hoseNum: " + this.usedNum + "," + this.housedNum);
-//		System.out.println("includedTime/extractTime: " + this.IncludedTime+ "," +this.exteactTime);
 		if (StringHandler.isAtLeastOneBlank(this.projectTitle,
 				this.projectCategory)) {
 			page.setResultSkip(this, true);
 			return;
 		}
-		
+
 		if (StringHandler.isAtLeastOneBlank(this.projectShortName,
 				this.projectCategory)) {
 			page.setResultSkip(this, true);
@@ -178,33 +180,33 @@ private String download_url;
 			page.setResultSkip(this, true);
 	}
 
-////////
-	
-public String getHomepage() {
-return homepage;
-}
+	// //////
 
-public void setHomepage(String homepage) {
-this.homepage = homepage;
-}
+	public String getHomepage() {
+		return homepage;
+	}
 
-public String getDoc_url() {
-return doc_url;
-}
+	public void setHomepage(String homepage) {
+		this.homepage = homepage;
+	}
 
-public void setDoc_url(String doc_url) {
-this.doc_url = doc_url;
-}
+	public String getDoc_url() {
+		return doc_url;
+	}
 
-public String getDownload_url() {
-return download_url;
-}
+	public void setDoc_url(String doc_url) {
+		this.doc_url = doc_url;
+	}
 
-public void setDownload_url(String download_url) {
-this.download_url = download_url;
-}
+	public String getDownload_url() {
+		return download_url;
+	}
 
-///////////
+	public void setDownload_url(String download_url) {
+		this.download_url = download_url;
+	}
+
+	// /////////
 
 	public String getProjectUrl() {
 		return projectUrl;

new_osseanextractor/src/main/java/net/trustie/model/sourceforge_Model.java

@@ -26,7 +26,7 @@ public class sourceforge_Model implements AfterExtractor, ValidateExtractor{
 	
 ///////////////
 	
-@ExtractBy("//*[@id='overview']/div[2]/section/p[3]/a/@href")
+@ExtractBy("//a[@id='homepage']/@href")
 
 private String homepage;
 //////////////////
@@ -362,8 +362,7 @@ private String homepage;
 	public void validate(Page page) {
 		this.name = this.name.trim();
 		//System.out.println("..." + this.name +"...");
-		if (StringHandler.isAtLeastOneBlank(this.name, this.platform,
-				this.categories, this.url)) {
+		if (StringHandler.isAtLeastOneBlank(this.name, this.homepage,this.url)) {
 			page.setResultSkip(this, true);
 			return;
 		}
@@ -372,11 +371,12 @@ private String homepage;
 			page.setResultSkip(this, true);
 			return;
 		}
-		;
+	
 		if (!DateHandler.canFormatToDate(this.lastUpdate, this.registeredTime,
 				this.collectTime)) {
 			page.setResultSkip(this, true);
 		}
+		
 
 	}
 

new_osseanextractor/src/main/java/net/trustie/one/Application.java

@@ -155,7 +155,7 @@ class ExtractThread implements Runnable{
 						
 						
 					}catch (Exception e){
-						System.out.println(e.getMessage());
+						e.printStackTrace();
 						pageErrorOutPut.returnErrorPage(page, e);
 					}					
 				}

new_osseanextractor/tasks.txt

@@ -1,18 +1 @@
-stackoverflow
-oschina_project
-iteye_blog
-csdn_blog
-freecode
-51cto_blog
-apache
-sourceforge
-openhub
-csdn_ask
-neitui
-lagou
-oschina_question
-csdn_topic
-dewen_question
-cnblogs_q_solved
-cnblogs_q_unsolved
-cnblogs_news
+sourceforge