diff --git a/project_match/bin/start_clear_table_befor_transfer.sh b/project_match/bin/start_clear_table_befor_transfer.sh new file mode 100644 index 000000000..eea21d05f --- /dev/null +++ b/project_match/bin/start_clear_table_befor_transfer.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +find ./target/classes -name "*.properties"|xargs rm -f +find ./target/classes -name "*.xml"|xargs rm -f +find ./target/classes -name "*.dic"|xargs rm -f + +#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib + +tmp='./target/classes':$tmp +tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp +tmp='./bin/resources':$tmp +CLASSPATH=$tmp:$CLASSPATH + + +echo $CLASSPATH +JVM_ARGS="-Xmn98m -Xmx512m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2" +#echo JVM_ARGS=$JVM_ARGS +#ulimit -n 400000 +#echo "" > nohup.out +java $JVM_ARGS -classpath $CLASSPATH com.ossean.ClearTableBeforTransfer >>log/clear_table_befor_transfer.log 2>&1 & \ No newline at end of file diff --git a/project_match/src/main/java/com/ossean/ClearTable.java b/project_match/src/main/java/com/ossean/ClearTable.java index f2762fbda..7a3054201 100644 --- a/project_match/src/main/java/com/ossean/ClearTable.java +++ b/project_match/src/main/java/com/ossean/ClearTable.java @@ -19,6 +19,7 @@ public class ClearTable { truncateTable("edd_pointers"); truncateTable("edd_relations"); truncateTable("synonymmings"); + truncateTable("open_source_projects"); deleteTaggings4Project(); logger.info("完成去重程序运行前数据表的清空和taggings表OpenSourceProject的删除"); diff --git a/project_match/src/main/java/com/ossean/ClearTableBeforTransfer.java b/project_match/src/main/java/com/ossean/ClearTableBeforTransfer.java new file mode 100644 index 000000000..c978faec3 --- /dev/null +++ b/project_match/src/main/java/com/ossean/ClearTableBeforTransfer.java @@ -0,0 +1,113 @@ +package com.ossean; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import org.apache.log4j.Logger; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.stereotype.Component; + + +@Component +public class ClearTableBeforTransfer { + + Logger logger = Logger.getLogger(ClearTableBeforTransfer.class); + public void start(){ + truncateTable("open_source_projects"); + deleteItemInEddPointers("edd_relations", "open_source_projects"); + deleteTaggings4Project(); + + logger.info("完成转移程序运行前数据表的清空和taggings表OpenSourceProject的删除"); + } + + public void truncateTable(String table){ + String sql = "truncate table " + table; + Connection conn = getConnection(); + if(conn == null){ + logger.info("没有获取到Connection"); + System.exit(-1); + } + PreparedStatement ps; + try { + ps = conn.prepareStatement(sql); + ps.execute(); + ps.close(); + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public void deleteTaggings4Project(){ + String sql = "delete from taggings where taggable_type=?"; + Connection conn = getConnection(); + if(conn == null){ + logger.info("没有获取到Connection"); + System.exit(-1); + } + try { + PreparedStatement ps = conn.prepareStatement(sql); + ps.setString(1, "OpenSourceProject"); + ps.execute(); + ps.close(); + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + //删除edd_pointers表中transfer对应的数据 + public void deleteItemInEddPointers(String sourceTableName, String targetTableName){ + String sql = "delete from edd_pointers where SourceTableName=? and TargetTableName=?"; + Connection conn = getConnection(); + if(conn == null){ + logger.info("没有获取到Connection"); + System.exit(-1); + } + try { + PreparedStatement ps = conn.prepareStatement(sql); + ps.setString(1, sourceTableName); + ps.setString(2, targetTableName); + ps.execute(); + ps.close(); + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public Connection getConnection(){ + Connection conn = null; + try { + Class.forName("com.mysql.jdbc.Driver").newInstance(); +// conn = DriverManager.getConnection("jdbc:mysql://172.16.128.30:3306/ossean_production", "gather", "influx1234"); + + conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/extract_result", "root", "123456"); + } catch (InstantiationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IllegalAccessException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ClassNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return conn; + } + + public static void main(String[] args){ + ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml"); + ClearTableBeforTransfer Main = applicationContext.getBean(ClearTableBeforTransfer.class); + Main.start(); + } +} diff --git a/project_match/src/main/java/com/ossean/util/TransferProjectsUtil.java b/project_match/src/main/java/com/ossean/util/TransferProjectsUtil.java index 467a43df0..3f5c6c812 100644 --- a/project_match/src/main/java/com/ossean/util/TransferProjectsUtil.java +++ b/project_match/src/main/java/com/ossean/util/TransferProjectsUtil.java @@ -145,6 +145,10 @@ public class TransferProjectsUtil { return null; OpenSourceProject result = new OpenSourceProject(); GatherProjectsModel hottestProject = getHottestProject(gatherProjectList); + if(hottestProject == null){ + //表示没有找到最热门的项目 只有freecode项目 + return null; + } int id = hottestProject.getId(); String name = hottestProject.getName(); String description = null; @@ -180,7 +184,7 @@ public class TransferProjectsUtil { } - if(!"FreeCode".equals(gatherProject.getSource())){ + if(!(("FreeCode".toLowerCase()).equals(gatherProject.getSource().toLowerCase()))){ //表示如果是FreeCode社区中的项目 只将tags属性合并 不进行下面的步骤 description = mergeTextWithSource(description, gatherProject.getDescription(), gatherProject.getSource()); // followers_num = hottestProject.getFollowers_num() == null ? 0 : hottestProject.getFollowers_num(); @@ -221,7 +225,7 @@ public class TransferProjectsUtil { result.setHomepage(homepage); result.setExtracted_time(extracted_time); result.setUpdate_mark(1);//表示是待更新的条目 - if("FreeCode".equals(result.getSource()) || result.getName() == null || result.getDescription() == null) + if(("FreeCode".toLowerCase()).equals(result.getSource().toLowerCase()) || result.getName() == null || result.getDescription() == null) return null; return result; @@ -287,8 +291,8 @@ public class TransferProjectsUtil { tmp_value = 2; else if(source.toLowerCase().equals("sourceforge")) tmp_value = 3; - else if(source.toLowerCase().equals("freecode")) - tmp_value = 4; +// else if(source.toLowerCase().equals("freecode")) //freecode社区中的项目不能成为最热项目 +// tmp_value = 4; else tmp_value = Integer.MAX_VALUE; if(head > tmp_value){ @@ -474,8 +478,8 @@ public class TransferProjectsUtil { continue;//认为项目相同 不替换 }else{ //删除osp_former - dbDest.deleteOpenSourceProjectsItem(openSourceProjectTableName,id); //级联删除了taggings表中对应的数据 - dbSource.deleteTaggingsByTaggableId(taggingTableName, id); + dbDest.deleteOpenSourceProjectsItem(openSourceProjectTableName,id); + dbSource.deleteTaggingsByTaggableId(taggingTableName, id); //由于没有级联删除 因此需要手动删除 } } } @@ -486,7 +490,7 @@ public class TransferProjectsUtil { * 比较两个osp对象的姓名是否相同 */ public boolean sameName(OpenSourceProject osp_former, OpenSourceProject osp){ - if(osp_former.getName().equals(osp.getName())) + if((osp_former.getName().toLowerCase()).equals(osp.getName().toLowerCase())) return true; else return false;