From 9cc5eefbf6e0097df491e22bcb3eefbd9f1d6ea1 Mon Sep 17 00:00:00 2001 From: nigel <609441862@qq.com> Date: Tue, 19 Apr 2016 15:57:16 +0800 Subject: [PATCH] add MergeProjectsByHomepageOnly --- .../start_merge_projects_by_homepage_only.sh | 20 ++ .../ossean/MergeProjectsByHomepageOnly.java | 282 ++++++++++++++++++ .../src/main/java/com/ossean/Test.java | 76 ++--- .../com/ossean/databaseSource/GatherDao.java | 1 + .../com/ossean/util/MergeProjectsUtil.java | 8 + .../resource/applicationContext-myBatis.xml | 2 +- 6 files changed, 354 insertions(+), 35 deletions(-) create mode 100644 project_match/bin/start_merge_projects_by_homepage_only.sh create mode 100644 project_match/src/main/java/com/ossean/MergeProjectsByHomepageOnly.java diff --git a/project_match/bin/start_merge_projects_by_homepage_only.sh b/project_match/bin/start_merge_projects_by_homepage_only.sh new file mode 100644 index 000000000..65272bc13 --- /dev/null +++ b/project_match/bin/start_merge_projects_by_homepage_only.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +find ./target/classes -name "*.properties"|xargs rm -f +find ./target/classes -name "*.xml"|xargs rm -f +find ./target/classes -name "*.dic"|xargs rm -f + +#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib + +tmp='./target/classes':$tmp +tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp +tmp='./bin/resources':$tmp +CLASSPATH=$tmp:$CLASSPATH + + +echo $CLASSPATH +JVM_ARGS="-Xmn98m -Xmx512m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2" +#echo JVM_ARGS=$JVM_ARGS +#ulimit -n 400000 +#echo "" > nohup.out +java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjectsByHomepageOnly >>log/MergeProjectsByHomepageOnly.log 2>&1 & \ No newline at end of file diff --git a/project_match/src/main/java/com/ossean/MergeProjectsByHomepageOnly.java b/project_match/src/main/java/com/ossean/MergeProjectsByHomepageOnly.java new file mode 100644 index 000000000..42cd996aa --- /dev/null +++ b/project_match/src/main/java/com/ossean/MergeProjectsByHomepageOnly.java @@ -0,0 +1,282 @@ +package com.ossean; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Resource; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.stereotype.Component; + +import com.ossean.databaseSource.DBSource; +import com.ossean.databaseSource.GatherDao; +import com.ossean.model.EddRelations; +import com.ossean.model.GatherProjectsModel; +import com.ossean.util.MergeProjectsUtil; + +@Component +public class MergeProjectsByHomepageOnly { + Logger logger = Logger.getLogger(this.getClass()); + + @Resource + private DBSource dbSource; + @Resource + private GatherDao gatherDao; + + @Qualifier("mergeProjectsUtil") + @Autowired + private MergeProjectsUtil mergeProjectsUtil; + + private static String pointerTableName = "edd_pointers"; + private static String sourceTableName = "gather_projects"; + private static String eddRelationTableName = "edd_relations"; + private static String targetTableName = "edd_relations_homepage_only"; + + public int readPointer(String table, String source, String target){ + int pointer = 1; + try { + pointer = dbSource.getPointer(table, source, target); + } catch(Exception e) { + logger.info("No such pointer! Create one"); + dbSource.insertPointer(table, source, target, pointer); + } + return pointer; + } + + public void start(){ + while(true){ + int maxId = dbSource.getMaxId(sourceTableName); + int startId1 = readPointer(pointerTableName, sourceTableName, targetTableName); + if(startId1 > maxId){ + logger.info("finish program! HAHAHA"); + System.exit(0); + } + //获取当前的项目 + GatherProjectsModel project = gatherDao.selectGPMById(sourceTableName, startId1); + + //获取当前项目相同homepage的GatherProject + //提取项目的homepage + String homepage = project.getHomepage(); + //如果是null 不需要查找相同homepage的项目 + List sameHomepageList = new ArrayList(); + if(homepage != null && !"".equals(homepage)){ + homepage = homepage.trim(); + //查找所有相同homepage字段的项目 + //判断homepage是否有http://或https:// + int index_http = homepage.indexOf("http://"); + int index_https = homepage.indexOf("https://"); + if(index_http == 0){ + homepage = homepage.substring(index_http + 7); //去除http:// + }else if(index_https == 0){ + homepage = homepage.substring(index_https + 8); //去除https:// + }else{ + //表示不是以http或https开头 + } + + if((homepage.lastIndexOf("/") == homepage.length()-1) && (homepage.length() != 0)){ + //如果homepage的最后一个字符是/ 需要去除掉 + homepage = homepage.substring(0, homepage.length() - 1); + } + if(homepage.length() > 0){ + //为homepage添加http或https头 + String homepage1 = "http://" + homepage; + String homepage2 = "https://" + homepage; + String homepage3 = "http://" + homepage + "/"; + String homepage4 = "https://" + homepage + "/"; + + sameHomepageList = gatherDao.selectGPMBySameHomePage(sourceTableName, homepage1, homepage2, homepage3, homepage4, project.getId(), project.getSource());//认为一定是相同的项目 + } + } + + //处理sameHomepageList 如果edd_relations表中有包含对应id的记录 则将所有的id糅合在一起，删除以前的记录，添加新的edd_relation记录 +// Map samePageMap = new HashMap(); + if(sameHomepageList.size() == 0){ + //表示没有找到对应homepage的项目 + //更新指针 + if(homepage == null){ + logger.info(project.getId() + "项目：" + project.getName() + " homepage为null"); + }else{ + logger.info(project.getId() + "项目：" + project.getName() + " 没有找到相同homepage的项目"); + } + + dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1); + continue; + } + + sameHomepageList.add(project); +// for(GatherProjectsModel model:sameHomepageList){ +// //记录所有的项目id +// String model_id = String.valueOf(model.getId()); +// samePageMap.put(model_id, 1); +// } +// + //根据新的匹配结果形成一个新的EddRelation对象 + EddRelations new_relation = new EddRelations(); + String new_gather_projects_ids = ","; + for(GatherProjectsModel model:sameHomepageList){ + new_gather_projects_ids = new_gather_projects_ids + model.getId() + ","; + } + new_relation.setGather_projects_ids(new_gather_projects_ids); + + //判断samePageMap中的记录在原记录中是不是已经存在了 + List resultInEddRelation = new ArrayList(); + for(GatherProjectsModel model:sameHomepageList){ + List relationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model.getId()); + for(EddRelations relation:relationList){ + resultInEddRelation.add(relation); + } + } + + //判断resultInEddRelation中最长的gather_projects_ids是不是包含了其他所有的结果项目 如果不是则需要将所有的项目id统计到一起并删除原来的所有结果 + //找到resultInEddRelation中最长的gather_projects_ids对应的对象 + EddRelations longestRelation = findLongestGatherProjectsIds(resultInEddRelation); + + //将new_relation插入resultInEddRelation中 + resultInEddRelation.add(new_relation); + + //判断最长的gather_projects_ids中是不是包括了所有其他eddrelation中匹配到的id + boolean hasOthers = containOtherRelations(longestRelation, resultInEddRelation); + + //如果hasOthers为true 表示确实包含所有其他的relation（包括新加入的relation） 因此不用重新组合插入新的relation + if(hasOthers == true){ + //删除除longestRelation以外所有数据库中的relation 其中new_relation对应的id为默认值0 可以一并删除（数据库中没有 但删除操作可以继续） + for(EddRelations relation:resultInEddRelation){ + if(relation.getId() != longestRelation.getId()){ + dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId()); + } + } + } + + //如果hasOthers为false 表示存在edd_relations对象中的项目id不包含在最长的gather_projects_ids中 需要重新组合所有id 并删除原来的edd_relations插入新的edd_relations + else{ +// EddRelations newEddRelation = combineNewEddRelation(resultInEddRelation); + String new_relation_string = mergeProjectsUtil.hasDiffProjFromSameCommunity(resultInEddRelation); + //判断所有id是不是都来自于不同的社区 如果是则插入newEddRelation 如果不是 则不进行操作 + if(new_relation_string == null){ + //表示有来自同一社区的项目，则不进行后续操作 + dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1); + continue; + } + EddRelations newEddRelation = new EddRelations(); + newEddRelation.setGather_projects_ids(new_relation_string); + + dbSource.insertEddRelations(eddRelationTableName, newEddRelation); + logger.info("插入新的edd_relations项，其中gather_projects_ids = " + newEddRelation.getGather_projects_ids()); + //删除之前的所有edd_relations记录 + for(EddRelations relation:resultInEddRelation){ + dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId()); + } + } + + dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1); + } + + } + + + private EddRelations findLongestGatherProjectsIds(List eddRelationList){ + EddRelations result = new EddRelations(); + int maxLength = 0; + for(EddRelations relation:eddRelationList){ + String gather_projects_ids = relation.getGather_projects_ids(); + int length = gather_projects_ids.split(",").length-1; + if(length > maxLength){ + maxLength = length; + result = relation; + } + } + return result; + } + + + private boolean containOtherRelations(EddRelations longestRelation, List relationList){ + boolean result = true; + for(EddRelations relation:relationList){ + if(containRelation(longestRelation, relation) == false){ + return false; + } + } + return result; + } + + private boolean containRelation(EddRelations longestRelation, EddRelations relation){ + boolean result = false; + String longestIds = longestRelation.getGather_projects_ids(); + String[] longestIdArray = longestIds.split(","); + //去除第一个空串元素 + longestIdArray = removeFirstElement(longestIdArray); + String[] relationIdArray = relation.getGather_projects_ids().split(","); + //去除第一个空串元素 + relationIdArray = removeFirstElement(relationIdArray); + //查看array2中的元素在array1中是否都存在 + result = containArray(longestIdArray, relationIdArray); + return result; + } + + private boolean containArray(String[] array1, String[] array2){ + boolean result = true; + for(String s2:array2){ + int i; + for(i = 0; i < array1.length; i++){ + String s1 = array1[i]; + if(s1.equals(s2)){ + break; //表示存在结果 匹配下一个string + } + } + if(i == array1.length){ + //表示匹配到最后都没有匹配到合适的结果 + return false; + } + } + return result; + } + + + //重新组合List中EddRelation对象对应的gather_projects_ids对应的id数据 + private EddRelations combineNewEddRelation(List eddRelationList){ + EddRelations result = new EddRelations(); + String new_gather_projects_ids = ","; + Map idMap = new HashMap(); + for(EddRelations relation:eddRelationList){ + String gather_projects_ids = relation.getGather_projects_ids(); + String[] idArray = gather_projects_ids.split(","); + for(String id:idArray){ + if(!"".equals(id)) + idMap.put(id, 1); + } + } + //根据map中的key形成new_gather_projects_ids + Set keys = idMap.keySet(); + Iterator it = keys.iterator(); + while(it.hasNext()){ + String key = it.next().toString(); + new_gather_projects_ids = new_gather_projects_ids + key + ","; + } + result.setGather_projects_ids(new_gather_projects_ids); + return result; + } + + + private String[] removeFirstElement(String[] array){ + String[] result = new String[array.length-1]; + for(int i = 1; i < array.length; i++){ + result[i-1] = array[i]; + } + return result; + } + + public static void main(String[] args){ + ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml"); + MergeProjectsByHomepageOnly Main = applicationContext.getBean(MergeProjectsByHomepageOnly.class); + Main.start(); + } + +} diff --git a/project_match/src/main/java/com/ossean/Test.java b/project_match/src/main/java/com/ossean/Test.java index e35ca6154..3bfe8e986 100644 --- a/project_match/src/main/java/com/ossean/Test.java +++ b/project_match/src/main/java/com/ossean/Test.java @@ -1,6 +1,11 @@ package com.ossean; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; import javax.annotation.Resource; @@ -9,6 +14,7 @@ import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.stereotype.Component; import com.ossean.databaseSource.GatherDao; +import com.ossean.model.EddRelations; import com.ossean.model.GatherProjectsModel; import com.ossean.util.MergeProjectsUtil; import com.ossean.util.StringHandler_ProjectName; @@ -25,6 +31,8 @@ public class Test { @Resource private GatherDao gatherDao; + private MergeProjectsByHomepageOnly mergeHomepage; + String synonyms = ",,,,,,"; @@ -42,49 +50,49 @@ public class Test { public void start(){ -// String result = util.mergeTextWithoutSource(synonyms, synonyms2, -1); -// System.out.println(result); + //测试combineNewEddRelations函数 + EddRelations relation1 = new EddRelations(); + EddRelations relation2 = new EddRelations(); + EddRelations relation3 = new EddRelations(); + String gather_projects_ids1 = ",1,2,3,4,"; + String gather_projects_ids2 = ",1,"; + String gather_projects_ids3 = ",3,4,5,6,"; -// String a = "http://"; -// a = a.substring(0 + 7); //去除http:// -// a.substring(0, a.length()-1); -// System.out.println(a); + String[] array1 = gather_projects_ids1.split(","); + String[] array2 = gather_projects_ids2.split(","); - GatherProjectsModel model1 = new GatherProjectsModel(); -// model1.setName(name); -// model1.setDescription(description); - model1.setTags(tag1); - model1.setHomepage(homepage); - model1.setSource(source1); +// boolean result = containArray(array1, array2); - GatherProjectsModel model2 = new GatherProjectsModel(); - model2.setTags(tag2); - model2.setSource(source2); + relation1.setGather_projects_ids(gather_projects_ids1); + relation2.setGather_projects_ids(gather_projects_ids2); + relation3.setGather_projects_ids(gather_projects_ids3); -// String result = StringHandler_ProjectName.getProjectWithoutComName("solr gnu"); + List relationList = new ArrayList(); + relationList.add(relation2); + relationList.add(relation3); + relationList.add(relation1); -// List result = mergeUtil.getSynonyms(model); -// -// for(String r:result){ -// System.out.println(r); -// } + EddRelations result = findLongestGatherProjectsIds(relationList); - if((""+null).equals("null")){ - System.out.println("插入数据"); - } - -// String name = "apache hadoop"; -// List result = gatherDao.selectGPMBySameName("gather_projects", name, 33); -// for(GatherProjectsModel model : result){ -// System.out.println("id is : " + model.getId()); -// } - -// double rate = mergeUtil.calSimilarityByTag(model1, model2); -// System.out.println(rate); -// mergeUtil.handleNewProject(model1, false); +// EddRelations result = combineNewEddRelation(relationList); +// System.out.println(result.getGather_projects_ids()); } + private EddRelations findLongestGatherProjectsIds(List eddRelationList){ + EddRelations result = new EddRelations(); + int maxLength = 0; + for(EddRelations relation:eddRelationList){ + String gather_projects_ids = relation.getGather_projects_ids(); + int length = gather_projects_ids.split(",").length-1; + if(length > maxLength){ + maxLength = length; + result = relation; + } + } + return result; + } + public static void main(String[] args){ ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml"); Test Main = applicationContext.getBean(Test.class); diff --git a/project_match/src/main/java/com/ossean/databaseSource/GatherDao.java b/project_match/src/main/java/com/ossean/databaseSource/GatherDao.java index 5c6e198ea..c7c4e9739 100644 --- a/project_match/src/main/java/com/ossean/databaseSource/GatherDao.java +++ b/project_match/src/main/java/com/ossean/databaseSource/GatherDao.java @@ -113,4 +113,5 @@ public interface GatherDao { //更新update_mark字段 @Select("update ${table} set update_mark=#{update_mark} where id=#{id}") public void updateUpdateMark(@Param("table") String table, @Param("update_mark") String update_mark, @Param("id") int id); + } \ No newline at end of file diff --git a/project_match/src/main/java/com/ossean/util/MergeProjectsUtil.java b/project_match/src/main/java/com/ossean/util/MergeProjectsUtil.java index dcaf34101..3dec95185 100644 --- a/project_match/src/main/java/com/ossean/util/MergeProjectsUtil.java +++ b/project_match/src/main/java/com/ossean/util/MergeProjectsUtil.java @@ -946,6 +946,9 @@ public class MergeProjectsUtil { sameHomepageList = gatherDao.selectGPMBySameHomePage(gatherProjectsTableName, homepage1, homepage2, homepage3, homepage4, model.getId(), model.getSource());//认为一定是相同的项目 + //查看sameHomepageList和当前的model存不存在来自同一社区的项目 如果存在则认为homepage匹配有问题 就将sameHomepageList清空 + + for(GatherProjectsModel m : sameHomepageList){ logger.info("project " + model.getId() + " and " + m.getId() + " match by homepage"); } @@ -1071,4 +1074,9 @@ public class MergeProjectsUtil { return result; } + + /** + * 判断是不是有来自同一社区的项目 + */ + } diff --git a/project_match/src/main/resource/applicationContext-myBatis.xml b/project_match/src/main/resource/applicationContext-myBatis.xml index 16be0526c..2605a2aa6 100644 --- a/project_match/src/main/resource/applicationContext-myBatis.xml +++ b/project_match/src/main/resource/applicationContext-myBatis.xml @@ -19,7 +19,7 @@ destroy-method="close"> + value="jdbc:mysql://localhost:3306/ossean_production_20160417?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />