kevinli
/
educoder
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add ClearTableBeforTransfer and change sth in transferProjects

This commit is contained in:
nigel 2016-04-14 09:02:55 +08:00
parent 591405d894
commit 7ac2778353
4 changed files with 145 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
project_match/bin/start_clear_table_befor_transfer.sh Normal file
View File

@ -0,0 +1,20 @@
#!/bin/bash
find ./target/classes -name "*.properties"|xargs rm -f
find ./target/classes -name "*.xml"|xargs rm -f
find ./target/classes -name "*.dic"|xargs rm -f
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
tmp='./target/classes':$tmp
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
tmp='./bin/resources':$tmp
CLASSPATH=$tmp:$CLASSPATH
echo $CLASSPATH
JVM_ARGS="-Xmn98m -Xmx512m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
#echo JVM_ARGS=$JVM_ARGS
#ulimit -n 400000
#echo "" > nohup.out
java $JVM_ARGS -classpath $CLASSPATH com.ossean.ClearTableBeforTransfer >>log/clear_table_befor_transfer.log 2>&1 &

1
project_match/src/main/java/com/ossean/ClearTable.java
View File

@ -19,6 +19,7 @@ public class ClearTable {
truncateTable("edd_pointers");
truncateTable("edd_relations");
truncateTable("synonymmings");
truncateTable("open_source_projects");
deleteTaggings4Project();
logger.info("完成去重程序运行前数据表的清空和taggings表OpenSourceProject的删除");

113
project_match/src/main/java/com/ossean/ClearTableBeforTransfer.java Normal file
View File

@ -0,0 +1,113 @@
package com.ossean;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import org.apache.log4j.Logger;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;
@Component
public class ClearTableBeforTransfer {
Logger logger = Logger.getLogger(ClearTableBeforTransfer.class);
public void start(){
truncateTable("open_source_projects");
deleteItemInEddPointers("edd_relations", "open_source_projects");
deleteTaggings4Project();
logger.info("完成转移程序运行前数据表的清空和taggings表OpenSourceProject的删除");
}
public void truncateTable(String table){
String sql = "truncate table " + table;
Connection conn = getConnection();
if(conn == null){
logger.info("没有获取到Connection");
System.exit(-1);
}
PreparedStatement ps;
try {
ps = conn.prepareStatement(sql);
ps.execute();
ps.close();
conn.close();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void deleteTaggings4Project(){
String sql = "delete from taggings where taggable_type=?";
Connection conn = getConnection();
if(conn == null){
logger.info("没有获取到Connection");
System.exit(-1);
}
try {
PreparedStatement ps = conn.prepareStatement(sql);
ps.setString(1, "OpenSourceProject");
ps.execute();
ps.close();
conn.close();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
//删除edd_pointers表中transfer对应的数据
public void deleteItemInEddPointers(String sourceTableName, String targetTableName){
String sql = "delete from edd_pointers where SourceTableName=? and TargetTableName=?";
Connection conn = getConnection();
if(conn == null){
logger.info("没有获取到Connection");
System.exit(-1);
}
try {
PreparedStatement ps = conn.prepareStatement(sql);
ps.setString(1, sourceTableName);
ps.setString(2, targetTableName);
ps.execute();
ps.close();
conn.close();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public Connection getConnection(){
Connection conn = null;
try {
Class.forName("com.mysql.jdbc.Driver").newInstance();
// conn = DriverManager.getConnection("jdbc:mysql://172.16.128.30:3306/ossean_production", "gather", "influx1234");
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/extract_result", "root", "123456");
} catch (InstantiationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return conn;
}
public static void main(String[] args){
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
ClearTableBeforTransfer Main = applicationContext.getBean(ClearTableBeforTransfer.class);
Main.start();
}
}

18
project_match/src/main/java/com/ossean/util/TransferProjectsUtil.java
View File

@ -145,6 +145,10 @@ public class TransferProjectsUtil {
return null;
OpenSourceProject result = new OpenSourceProject();
GatherProjectsModel hottestProject = getHottestProject(gatherProjectList);
if(hottestProject == null){
//表示没有找到最热门的项目 只有freecode项目
return null;
}
int id = hottestProject.getId();
String name = hottestProject.getName();
String description = null;
@ -180,7 +184,7 @@ public class TransferProjectsUtil {
}
if(!"FreeCode".equals(gatherProject.getSource())){
if(!(("FreeCode".toLowerCase()).equals(gatherProject.getSource().toLowerCase()))){
//表示如果是FreeCode社区中的项目 只将tags属性合并 不进行下面的步骤
description = mergeTextWithSource(description, gatherProject.getDescription(), gatherProject.getSource());
// followers_num = hottestProject.getFollowers_num() == null ? 0 : hottestProject.getFollowers_num();
@ -221,7 +225,7 @@ public class TransferProjectsUtil {
result.setHomepage(homepage);
result.setExtracted_time(extracted_time);
result.setUpdate_mark(1);//表示是待更新的条目
if("FreeCode".equals(result.getSource()) || result.getName() == null || result.getDescription() == null)
if(("FreeCode".toLowerCase()).equals(result.getSource().toLowerCase()) || result.getName() == null || result.getDescription() == null)
return null;
return result;
@ -287,8 +291,8 @@ public class TransferProjectsUtil {
tmp_value = 2;
else if(source.toLowerCase().equals("sourceforge"))
tmp_value = 3;
else if(source.toLowerCase().equals("freecode"))
tmp_value = 4;
// else if(source.toLowerCase().equals("freecode")) //freecode社区中的项目不能成为最热项目
// tmp_value = 4;
else
tmp_value = Integer.MAX_VALUE;
if(head > tmp_value){
@ -474,8 +478,8 @@ public class TransferProjectsUtil {
continue;//认为项目相同 不替换
}else{
//删除osp_former
dbDest.deleteOpenSourceProjectsItem(openSourceProjectTableName,id); //级联删除了taggings表中对应的数据
dbSource.deleteTaggingsByTaggableId(taggingTableName, id);
dbDest.deleteOpenSourceProjectsItem(openSourceProjectTableName,id);
dbSource.deleteTaggingsByTaggableId(taggingTableName, id); //由于没有级联删除 因此需要手动删除
}
}
}
@ -486,7 +490,7 @@ public class TransferProjectsUtil {
* 比较两个osp对象的姓名是否相同
*/
public boolean sameName(OpenSourceProject osp_former, OpenSourceProject osp){
if(osp_former.getName().equals(osp.getName()))
if((osp_former.getName().toLowerCase()).equals(osp.getName().toLowerCase()))
return true;
else
return false;