add MergeProjectsByHomepageOnly
This commit is contained in:
parent
ab524d69a2
commit
9cc5eefbf6
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
tmp='./bin/resources':$tmp
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn98m -Xmx512m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjectsByHomepageOnly >>log/MergeProjectsByHomepageOnly.log 2>&1 &
|
||||
|
|
@ -0,0 +1,282 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.MergeProjectsUtil;
|
||||
|
||||
@Component
|
||||
public class MergeProjectsByHomepageOnly {
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
|
||||
@Qualifier("mergeProjectsUtil")
|
||||
@Autowired
|
||||
private MergeProjectsUtil mergeProjectsUtil;
|
||||
|
||||
private static String pointerTableName = "edd_pointers";
|
||||
private static String sourceTableName = "gather_projects";
|
||||
private static String eddRelationTableName = "edd_relations";
|
||||
private static String targetTableName = "edd_relations_homepage_only";
|
||||
|
||||
public int readPointer(String table, String source, String target){
|
||||
int pointer = 1;
|
||||
try {
|
||||
pointer = dbSource.getPointer(table, source, target);
|
||||
} catch(Exception e) {
|
||||
logger.info("No such pointer! Create one");
|
||||
dbSource.insertPointer(table, source, target, pointer);
|
||||
}
|
||||
return pointer;
|
||||
}
|
||||
|
||||
public void start(){
|
||||
while(true){
|
||||
int maxId = dbSource.getMaxId(sourceTableName);
|
||||
int startId1 = readPointer(pointerTableName, sourceTableName, targetTableName);
|
||||
if(startId1 > maxId){
|
||||
logger.info("finish program! HAHAHA");
|
||||
System.exit(0);
|
||||
}
|
||||
//获取当前的项目
|
||||
GatherProjectsModel project = gatherDao.selectGPMById(sourceTableName, startId1);
|
||||
|
||||
//获取当前项目相同homepage的GatherProject
|
||||
//提取项目的homepage
|
||||
String homepage = project.getHomepage();
|
||||
//如果是null 不需要查找相同homepage的项目
|
||||
List<GatherProjectsModel> sameHomepageList = new ArrayList<GatherProjectsModel>();
|
||||
if(homepage != null && !"".equals(homepage)){
|
||||
homepage = homepage.trim();
|
||||
//查找所有相同homepage字段的项目
|
||||
//判断homepage是否有http://或https://
|
||||
int index_http = homepage.indexOf("http://");
|
||||
int index_https = homepage.indexOf("https://");
|
||||
if(index_http == 0){
|
||||
homepage = homepage.substring(index_http + 7); //去除http://
|
||||
}else if(index_https == 0){
|
||||
homepage = homepage.substring(index_https + 8); //去除https://
|
||||
}else{
|
||||
//表示不是以http或https开头
|
||||
}
|
||||
|
||||
if((homepage.lastIndexOf("/") == homepage.length()-1) && (homepage.length() != 0)){
|
||||
//如果homepage的最后一个字符是/ 需要去除掉
|
||||
homepage = homepage.substring(0, homepage.length() - 1);
|
||||
}
|
||||
if(homepage.length() > 0){
|
||||
//为homepage添加http或https头
|
||||
String homepage1 = "http://" + homepage;
|
||||
String homepage2 = "https://" + homepage;
|
||||
String homepage3 = "http://" + homepage + "/";
|
||||
String homepage4 = "https://" + homepage + "/";
|
||||
|
||||
sameHomepageList = gatherDao.selectGPMBySameHomePage(sourceTableName, homepage1, homepage2, homepage3, homepage4, project.getId(), project.getSource());//认为一定是相同的项目
|
||||
}
|
||||
}
|
||||
|
||||
//处理sameHomepageList 如果edd_relations表中有包含对应id的记录 则将所有的id糅合在一起，删除以前的记录，添加新的edd_relation记录
|
||||
// Map<String, Integer> samePageMap = new HashMap<String, Integer>();
|
||||
if(sameHomepageList.size() == 0){
|
||||
//表示没有找到对应homepage的项目
|
||||
//更新指针
|
||||
if(homepage == null){
|
||||
logger.info(project.getId() + "项目：" + project.getName() + " homepage为null");
|
||||
}else{
|
||||
logger.info(project.getId() + "项目：" + project.getName() + " 没有找到相同homepage的项目");
|
||||
}
|
||||
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
sameHomepageList.add(project);
|
||||
// for(GatherProjectsModel model:sameHomepageList){
|
||||
// //记录所有的项目id
|
||||
// String model_id = String.valueOf(model.getId());
|
||||
// samePageMap.put(model_id, 1);
|
||||
// }
|
||||
//
|
||||
//根据新的匹配结果形成一个新的EddRelation对象
|
||||
EddRelations new_relation = new EddRelations();
|
||||
String new_gather_projects_ids = ",";
|
||||
for(GatherProjectsModel model:sameHomepageList){
|
||||
new_gather_projects_ids = new_gather_projects_ids + model.getId() + ",";
|
||||
}
|
||||
new_relation.setGather_projects_ids(new_gather_projects_ids);
|
||||
|
||||
//判断samePageMap中的记录在原记录中是不是已经存在了
|
||||
List<EddRelations> resultInEddRelation = new ArrayList<EddRelations>();
|
||||
for(GatherProjectsModel model:sameHomepageList){
|
||||
List<EddRelations> relationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model.getId());
|
||||
for(EddRelations relation:relationList){
|
||||
resultInEddRelation.add(relation);
|
||||
}
|
||||
}
|
||||
|
||||
//判断resultInEddRelation中最长的gather_projects_ids是不是包含了其他所有的结果项目 如果不是则需要将所有的项目id统计到一起并删除原来的所有结果
|
||||
//找到resultInEddRelation中最长的gather_projects_ids对应的对象
|
||||
EddRelations longestRelation = findLongestGatherProjectsIds(resultInEddRelation);
|
||||
|
||||
//将new_relation插入resultInEddRelation中
|
||||
resultInEddRelation.add(new_relation);
|
||||
|
||||
//判断最长的gather_projects_ids中是不是包括了所有其他eddrelation中匹配到的id
|
||||
boolean hasOthers = containOtherRelations(longestRelation, resultInEddRelation);
|
||||
|
||||
//如果hasOthers为true 表示确实包含所有其他的relation（包括新加入的relation） 因此不用重新组合插入新的relation
|
||||
if(hasOthers == true){
|
||||
//删除除longestRelation以外所有数据库中的relation 其中new_relation对应的id为默认值0 可以一并删除（数据库中没有 但删除操作可以继续）
|
||||
for(EddRelations relation:resultInEddRelation){
|
||||
if(relation.getId() != longestRelation.getId()){
|
||||
dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//如果hasOthers为false 表示存在edd_relations对象中的项目id不包含在最长的gather_projects_ids中 需要重新组合所有id 并删除原来的edd_relations插入新的edd_relations
|
||||
else{
|
||||
// EddRelations newEddRelation = combineNewEddRelation(resultInEddRelation);
|
||||
String new_relation_string = mergeProjectsUtil.hasDiffProjFromSameCommunity(resultInEddRelation);
|
||||
//判断所有id是不是都来自于不同的社区 如果是则插入newEddRelation 如果不是 则不进行操作
|
||||
if(new_relation_string == null){
|
||||
//表示有来自同一社区的项目，则不进行后续操作
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1);
|
||||
continue;
|
||||
}
|
||||
EddRelations newEddRelation = new EddRelations();
|
||||
newEddRelation.setGather_projects_ids(new_relation_string);
|
||||
|
||||
dbSource.insertEddRelations(eddRelationTableName, newEddRelation);
|
||||
logger.info("插入新的edd_relations项，其中gather_projects_ids = " + newEddRelation.getGather_projects_ids());
|
||||
//删除之前的所有edd_relations记录
|
||||
for(EddRelations relation:resultInEddRelation){
|
||||
dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId());
|
||||
}
|
||||
}
|
||||
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private EddRelations findLongestGatherProjectsIds(List<EddRelations> eddRelationList){
|
||||
EddRelations result = new EddRelations();
|
||||
int maxLength = 0;
|
||||
for(EddRelations relation:eddRelationList){
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
int length = gather_projects_ids.split(",").length-1;
|
||||
if(length > maxLength){
|
||||
maxLength = length;
|
||||
result = relation;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private boolean containOtherRelations(EddRelations longestRelation, List<EddRelations> relationList){
|
||||
boolean result = true;
|
||||
for(EddRelations relation:relationList){
|
||||
if(containRelation(longestRelation, relation) == false){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean containRelation(EddRelations longestRelation, EddRelations relation){
|
||||
boolean result = false;
|
||||
String longestIds = longestRelation.getGather_projects_ids();
|
||||
String[] longestIdArray = longestIds.split(",");
|
||||
//去除第一个空串元素
|
||||
longestIdArray = removeFirstElement(longestIdArray);
|
||||
String[] relationIdArray = relation.getGather_projects_ids().split(",");
|
||||
//去除第一个空串元素
|
||||
relationIdArray = removeFirstElement(relationIdArray);
|
||||
//查看array2中的元素在array1中是否都存在
|
||||
result = containArray(longestIdArray, relationIdArray);
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean containArray(String[] array1, String[] array2){
|
||||
boolean result = true;
|
||||
for(String s2:array2){
|
||||
int i;
|
||||
for(i = 0; i < array1.length; i++){
|
||||
String s1 = array1[i];
|
||||
if(s1.equals(s2)){
|
||||
break; //表示存在结果 匹配下一个string
|
||||
}
|
||||
}
|
||||
if(i == array1.length){
|
||||
//表示匹配到最后都没有匹配到合适的结果
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
//重新组合List<EddRelations>中EddRelation对象对应的gather_projects_ids对应的id数据
|
||||
private EddRelations combineNewEddRelation(List<EddRelations> eddRelationList){
|
||||
EddRelations result = new EddRelations();
|
||||
String new_gather_projects_ids = ",";
|
||||
Map<String, Integer> idMap = new HashMap<String, Integer>();
|
||||
for(EddRelations relation:eddRelationList){
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
String[] idArray = gather_projects_ids.split(",");
|
||||
for(String id:idArray){
|
||||
if(!"".equals(id))
|
||||
idMap.put(id, 1);
|
||||
}
|
||||
}
|
||||
//根据map中的key形成new_gather_projects_ids
|
||||
Set<String> keys = idMap.keySet();
|
||||
Iterator it = keys.iterator();
|
||||
while(it.hasNext()){
|
||||
String key = it.next().toString();
|
||||
new_gather_projects_ids = new_gather_projects_ids + key + ",";
|
||||
}
|
||||
result.setGather_projects_ids(new_gather_projects_ids);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private String[] removeFirstElement(String[] array){
|
||||
String[] result = new String[array.length-1];
|
||||
for(int i = 1; i < array.length; i++){
|
||||
result[i-1] = array[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
MergeProjectsByHomepageOnly Main = applicationContext.getBean(MergeProjectsByHomepageOnly.class);
|
||||
Main.start();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,6 +1,11 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
|
|
@ -9,6 +14,7 @@ import org.springframework.context.support.ClassPathXmlApplicationContext;
|
|||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.MergeProjectsUtil;
|
||||
import com.ossean.util.StringHandler_ProjectName;
|
||||
|
|
@ -25,6 +31,8 @@ public class Test {
|
|||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
|
||||
private MergeProjectsByHomepageOnly mergeHomepage;
|
||||
|
||||
|
||||
|
||||
String synonyms = "<maven>,<maven>,<hello>,<shit>,<finally>,<shit>,";
|
||||
|
|
@ -42,49 +50,49 @@ public class Test {
|
|||
|
||||
|
||||
public void start(){
|
||||
// String result = util.mergeTextWithoutSource(synonyms, synonyms2, -1);
|
||||
// System.out.println(result);
|
||||
//测试combineNewEddRelations函数
|
||||
EddRelations relation1 = new EddRelations();
|
||||
EddRelations relation2 = new EddRelations();
|
||||
EddRelations relation3 = new EddRelations();
|
||||
String gather_projects_ids1 = ",1,2,3,4,";
|
||||
String gather_projects_ids2 = ",1,";
|
||||
String gather_projects_ids3 = ",3,4,5,6,";
|
||||
|
||||
// String a = "http://";
|
||||
// a = a.substring(0 + 7); //去除http://
|
||||
// a.substring(0, a.length()-1);
|
||||
// System.out.println(a);
|
||||
String[] array1 = gather_projects_ids1.split(",");
|
||||
String[] array2 = gather_projects_ids2.split(",");
|
||||
|
||||
GatherProjectsModel model1 = new GatherProjectsModel();
|
||||
// model1.setName(name);
|
||||
// model1.setDescription(description);
|
||||
model1.setTags(tag1);
|
||||
model1.setHomepage(homepage);
|
||||
model1.setSource(source1);
|
||||
// boolean result = containArray(array1, array2);
|
||||
|
||||
GatherProjectsModel model2 = new GatherProjectsModel();
|
||||
model2.setTags(tag2);
|
||||
model2.setSource(source2);
|
||||
relation1.setGather_projects_ids(gather_projects_ids1);
|
||||
relation2.setGather_projects_ids(gather_projects_ids2);
|
||||
relation3.setGather_projects_ids(gather_projects_ids3);
|
||||
|
||||
// String result = StringHandler_ProjectName.getProjectWithoutComName("solr gnu");
|
||||
List<EddRelations> relationList = new ArrayList<EddRelations>();
|
||||
relationList.add(relation2);
|
||||
relationList.add(relation3);
|
||||
relationList.add(relation1);
|
||||
|
||||
// List<String> result = mergeUtil.getSynonyms(model);
|
||||
//
|
||||
// for(String r:result){
|
||||
// System.out.println(r);
|
||||
// }
|
||||
EddRelations result = findLongestGatherProjectsIds(relationList);
|
||||
|
||||
if((""+null).equals("null")){
|
||||
System.out.println("插入数据");
|
||||
}
|
||||
|
||||
// String name = "apache hadoop";
|
||||
// List<GatherProjectsModel> result = gatherDao.selectGPMBySameName("gather_projects", name, 33);
|
||||
// for(GatherProjectsModel model : result){
|
||||
// System.out.println("id is : " + model.getId());
|
||||
// }
|
||||
|
||||
// double rate = mergeUtil.calSimilarityByTag(model1, model2);
|
||||
// System.out.println(rate);
|
||||
// mergeUtil.handleNewProject(model1, false);
|
||||
// EddRelations result = combineNewEddRelation(relationList);
|
||||
// System.out.println(result.getGather_projects_ids());
|
||||
}
|
||||
|
||||
|
||||
private EddRelations findLongestGatherProjectsIds(List<EddRelations> eddRelationList){
|
||||
EddRelations result = new EddRelations();
|
||||
int maxLength = 0;
|
||||
for(EddRelations relation:eddRelationList){
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
int length = gather_projects_ids.split(",").length-1;
|
||||
if(length > maxLength){
|
||||
maxLength = length;
|
||||
result = relation;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
Test Main = applicationContext.getBean(Test.class);
|
||||
|
|
|
|||
|
|
@ -113,4 +113,5 @@ public interface GatherDao {
|
|||
//更新update_mark字段
|
||||
@Select("update ${table} set update_mark=#{update_mark} where id=#{id}")
|
||||
public void updateUpdateMark(@Param("table") String table, @Param("update_mark") String update_mark, @Param("id") int id);
|
||||
|
||||
}
|
||||
|
|
@ -946,6 +946,9 @@ public class MergeProjectsUtil {
|
|||
|
||||
sameHomepageList = gatherDao.selectGPMBySameHomePage(gatherProjectsTableName, homepage1, homepage2, homepage3, homepage4, model.getId(), model.getSource());//认为一定是相同的项目
|
||||
|
||||
//查看sameHomepageList和当前的model存不存在来自同一社区的项目 如果存在则认为homepage匹配有问题 就将sameHomepageList清空
|
||||
|
||||
|
||||
for(GatherProjectsModel m : sameHomepageList){
|
||||
logger.info("project " + model.getId() + " and " + m.getId() + " match by homepage");
|
||||
}
|
||||
|
|
@ -1071,4 +1074,9 @@ public class MergeProjectsUtil {
|
|||
return result;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 判断是不是有来自同一社区的项目
|
||||
*/
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost:3306/extract_result?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
value="jdbc:mysql://localhost:3306/ossean_production_20160417?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
|
|
|
|||
Loading…
Reference in New Issue