1101 lines
39 KiB
Java
1101 lines
39 KiB
Java
package com.ossean.util;
|
||
|
||
import java.io.IOException;
|
||
import java.io.Reader;
|
||
import java.io.StringReader;
|
||
import java.util.ArrayList;
|
||
import java.util.HashMap;
|
||
import java.util.Iterator;
|
||
import java.util.List;
|
||
import java.util.Map;
|
||
import java.util.Set;
|
||
|
||
import javax.annotation.Resource;
|
||
|
||
import org.apache.log4j.Logger;
|
||
import org.springframework.beans.factory.annotation.Autowired;
|
||
import org.springframework.beans.factory.annotation.Qualifier;
|
||
import org.springframework.stereotype.Component;
|
||
import org.springframework.transaction.annotation.Propagation;
|
||
import org.springframework.transaction.annotation.Transactional;
|
||
import org.wltea.analyzer.core.IKSegmenter;
|
||
import org.wltea.analyzer.core.Lexeme;
|
||
|
||
import com.ossean.databaseDest.DBDest;
|
||
import com.ossean.databaseSource.DBSource;
|
||
import com.ossean.databaseSource.GatherDao;
|
||
import com.ossean.model.EddRelations;
|
||
import com.ossean.model.GatherProjectsModel;
|
||
import com.ossean.model.Synonymmings;
|
||
import com.ossean.model.Synonyms;
|
||
import com.ossean.model.Taggings;
|
||
import com.ossean.model.Tags;
|
||
|
||
@Component("mergeProjectsUtil")
|
||
public class MergeProjectsUtil {
|
||
Logger logger = Logger.getLogger(this.getClass());
|
||
@Resource
|
||
private DBSource dbSource;
|
||
@Resource
|
||
private GatherDao gatherDao;
|
||
@Resource
|
||
private DBDest dbDest;
|
||
@Qualifier("gatherProjectsUtil")
|
||
@Autowired
|
||
private GatherProjectsUtil gatherProjectsUtil;
|
||
|
||
private String synonymTableName = "synonyms";
|
||
private String synonymmingTableName = "synonymmings";
|
||
private String gatherProjectsTableName = "gather_projects";
|
||
private String pointerTableName = "edd_pointers";
|
||
|
||
private String taggingsTableName = "taggings";
|
||
private String tagsTableName = "tags";
|
||
|
||
private String eddRelationTableName = "edd_relations";
|
||
|
||
|
||
//判断两个项目是否相同 默认两个项目的urlMD5码是不同的
|
||
public boolean areTwoProjectsTheSame(GatherProjectsModel model1, GatherProjectsModel model2){
|
||
/**
|
||
* 如果是同一个社区 则默认不相同
|
||
*/
|
||
if(model1.getSource().equals(model2.getSource()))
|
||
return false;
|
||
boolean result = false;
|
||
//两个项目初步匹配结果是相同的
|
||
boolean stepOneResult = false;
|
||
result = isTheSameAfterTFIDF(model1, model2, stepOneResult);
|
||
return result;
|
||
}
|
||
|
||
|
||
//edd算法提取linkNames 其中包含项目名本身
|
||
public List<String> getSynonyms(GatherProjectsModel model){
|
||
|
||
model.setName(model.getName().trim());
|
||
model.setDescription(model.getDescription().trim());
|
||
|
||
List<String> result = dbSource.getSynonyms(synonymTableName, synonymmingTableName, model.getId());//表示当前的项目已经处理过了 当前的是需要更新的
|
||
if(result.size() != 0){
|
||
for(String s : result){
|
||
logger.info("gather_project:" + model.getId() + " gets synonyms:" + s + " in table");
|
||
}
|
||
return result; //表示当前项目的别名在数据库中已经存在了
|
||
}
|
||
String description = model.getDescription();
|
||
if(description == null)
|
||
return result;
|
||
|
||
Map<String,Integer> linkNames = new HashMap<String,Integer>();//用于存储所有提取出来的关联项目名
|
||
//对项目名进行处理
|
||
//提取项目名中去除公司名或基金组织的信息
|
||
String projectNameWithoutComName = StringHandler_ProjectName.getProjectWithoutComName(model.getName());
|
||
if(!"".equals(projectNameWithoutComName)){
|
||
//表示匹配到了相应公司或基金委的项目
|
||
linkNames.put(projectNameWithoutComName, 1);
|
||
}
|
||
|
||
//提取项目名括号中的信息
|
||
String bracket = StringHandler_ProjectName.getBracket(model.getName());
|
||
if(!"".equals(bracket)){
|
||
//去除括号中信息的特殊符号
|
||
bracket = RegexHandler.extractEngDecChiAndDot(bracket);
|
||
if(!RegexHandler.onlySpecialChar(bracket)){
|
||
System.out.println("项目括号中的信息： "+bracket);
|
||
linkNames.put(bracket, 1);
|
||
}
|
||
}
|
||
|
||
//读取be动词表
|
||
List<String> beWords = FileReader.read("./files/beVerb.txt");
|
||
description = StringHandler.getFirstSentence(description);
|
||
for(String beWord:beWords){
|
||
//按系动词表优先级进行匹配
|
||
String linkName = StringHandler.findLinkName(description, beWord);
|
||
System.out.println("desc be动词之前的： "+linkName);
|
||
if(linkName == null){
|
||
//表示没有匹配到当前的系动词 进行下一个匹配
|
||
continue;
|
||
}
|
||
//特征短语去噪
|
||
//提取主副描述信息(括号中)
|
||
List<String> linkNameResult = EDDHandler.getMainAndViceDescriptionByBracket(linkName);
|
||
//提取主副描述信息(known as+连词)
|
||
linkNameResult = EDDHandler.getMainAndViceDescriptionByKnowAs(linkNameResult);
|
||
//去除定语描述信息(逗号定语)
|
||
linkNameResult = EDDHandler.removeComma(linkNameResult);
|
||
//去除定语描述信息(定冠词The)
|
||
linkNameResult = EDDHandler.removeDemonstrativeWords(linkNameResult);
|
||
//去除从句描述信息
|
||
linkNameResult = EDDHandler.removeArrtibutiveClause(linkNameResult);
|
||
//去除指示代词
|
||
linkNameResult = EDDHandler.removePronoun(linkNameResult);
|
||
//去除项目常用词
|
||
linkNameResult = EDDHandler.removeProject(linkNameResult);
|
||
//去除指示代词和项目常用词两者的笛卡尔积
|
||
linkNameResult = EDDHandler.removePPCombine(linkNameResult);
|
||
// //特殊字符处理(空字符串)
|
||
// linkNameResult = EDDHandler.removeEmpty(linkNameResult);
|
||
// //特殊字符处理(引号)
|
||
// linkNameResult = EDDHandler.removeSemicolon(linkNameResult);
|
||
// 特殊字符处理(除英文中文外 中间用空格连接)
|
||
// linkNameResult = EDDHandler.removeSingleColon(linkNameResult);
|
||
//提取项目中
|
||
linkNameResult = RegexHandler.extractEngDecChiAndDot(linkNameResult);
|
||
//删除只有特殊字符的同义词
|
||
linkNameResult = RegexHandler.removeOnlySpecial(linkNameResult);
|
||
for(String name:linkNameResult){
|
||
if(linkNames.containsKey(name)){
|
||
linkNames.put(name, linkNames.get(name)+1);
|
||
}else{
|
||
if(!name.equals(model.getName().toLowerCase()))
|
||
linkNames.put(name, 1);
|
||
}
|
||
}
|
||
break;
|
||
}//遍历be动词 for循环结尾
|
||
|
||
//将map转换成List
|
||
Set<String> keys = linkNames.keySet();
|
||
Iterator<String> it = keys.iterator();
|
||
while(it.hasNext()){
|
||
String a = it.next();
|
||
result.add(a);
|
||
System.out.println("result中包含的： "+a);
|
||
}
|
||
|
||
/**
|
||
* 对上面提取出来的同义词结果进行进一步限制 返回结果为result3
|
||
*/
|
||
List<String> result2 = new ArrayList<String>();
|
||
//若项目名为缩写形式 项目描述信息中存不存在项目名的全称
|
||
|
||
String fullName = StringHandler.getFullName(model.getName().toLowerCase(), result);
|
||
if(!fullName.equals("")){
|
||
result2.add(fullName.toLowerCase());//表示项目名确实是缩写 并提取到了项目全名
|
||
System.out.println("项目的全称为： "+fullName);
|
||
}
|
||
|
||
|
||
String shortName = StringHandler.getShortName(model.getName().toLowerCase(), result);
|
||
if(!shortName.equals("")){
|
||
result2.add(shortName.toLowerCase());//表示项目名确实是全称 并提取到了项目缩写
|
||
System.out.println("项目的简称为： "+shortName);
|
||
}
|
||
|
||
//要求描述信息中出现的项目名对应的词 按照顺序链接到一起
|
||
for(int i = 0; i < result.size(); i++){
|
||
//同义词要在项目名中出现
|
||
String extract = "";
|
||
String synonym = result.get(i);
|
||
String[] words = synonym.split(" ");//按照空格进行分词
|
||
//去掉括号检测同义词是否在项目名中出现
|
||
String modelNameWithoutBracket = StringHandler_ProjectName.removeBracket(model.getName().toLowerCase());
|
||
System.out.println("modelNameWithoutBracket : "+modelNameWithoutBracket);
|
||
for(String word:words){
|
||
//查看每个词在项目名中是否出现
|
||
if(modelNameWithoutBracket.indexOf(word) >= 0){
|
||
//表示项目名中存在这个分词
|
||
extract += word + " ";
|
||
}
|
||
}
|
||
if(!"".equals(extract)){
|
||
//表示确实是别名
|
||
extract = extract.substring(0, extract.length() - 1);
|
||
System.out.println("别名在项目名中有出现： "+extract);
|
||
result2.add(extract);
|
||
continue;
|
||
}
|
||
}
|
||
|
||
//对最终结果进行处理：如果与项目原名相同：去除 如果提取出了相同的结果：去除重复的
|
||
Map<String,Integer> tmp = new HashMap<String,Integer>();
|
||
for(String r:result2){
|
||
r = r.toLowerCase();
|
||
if(!r.equals(model.getName().toLowerCase()))
|
||
tmp.put(r, 1);
|
||
}
|
||
//将map中的key转换为list
|
||
List<String> result3 = new ArrayList<String>();
|
||
Set<String> keys_tmp = tmp.keySet();
|
||
Iterator it_tmp = keys_tmp.iterator();
|
||
while(it_tmp.hasNext())
|
||
result3.add(it_tmp.next().toString());
|
||
result3.add(model.getName().toLowerCase());
|
||
//项目同义词不包括项目原名
|
||
|
||
|
||
//输出日志文件
|
||
for(String s : result3){
|
||
logger.info("gather_project:" + model.getId() + " gets synonyms:" + s + " by description");
|
||
}
|
||
return result3;
|
||
}
|
||
|
||
|
||
//根据项目别名判断两个项目初步匹配是否正确 返回值为匹配上的项目别名
|
||
public boolean isTheSameByLinkName(List<String> linkNames1, List<String> linkNames2){
|
||
//对两个项目的别名之间进行匹配
|
||
for(int i = 0; i < linkNames1.size(); i++){
|
||
String linkName1 = linkNames1.get(i);
|
||
for(int j = 0; j < linkNames2.size(); j++){
|
||
String linkName2 = linkNames2.get(j);
|
||
if(linkName1.equals(linkName2))
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
|
||
//去伪取真过程
|
||
public boolean isTheSameAfterTFIDF(GatherProjectsModel model1, GatherProjectsModel model2, boolean stepOneResult){
|
||
//获取两个对象的tags属性
|
||
String tags1 = model1.getTags();
|
||
String tags2 = model2.getTags();
|
||
if((tags1 != null && tags2 != null)&&(!"".equals(tags1) && !"".equals(tags2))){
|
||
//表示两个项目都有标签
|
||
double similarity = calSimilarityByTag(model1, model2);
|
||
System.out.println("calSimilarityByTag"+String.valueOf(similarity));
|
||
if(stepOneResult){
|
||
//表示两个项目源自一个社区 去伪
|
||
//如果两个项目匹配结果 weight>0.9 表示是同一个项目
|
||
if(similarity > 0.9)
|
||
return true;
|
||
return false;
|
||
}else{
|
||
//表示两个项目不来自同一个社区 存真
|
||
//如果两个项目匹配结果 weight<0.1 表示不是同一个项目
|
||
if(similarity < 0.1)
|
||
return false;
|
||
return true;
|
||
}
|
||
}
|
||
else{
|
||
if((tags1 == null && tags2 != null) || ("".equals(tags1) && !"".equals(tags2)) || (!"".equals(tags1) && "".equals(tags2)) || (tags1 != null && tags2 == null)){
|
||
//表示两者有一个项目有标签
|
||
Map<String, Integer> result = new HashMap<String, Integer>();
|
||
List<String> tagList = new ArrayList<String>();
|
||
int count_tag = 0;
|
||
if(tags1 == null){
|
||
//表示项目2有标签
|
||
//对项目1的描述信息进行分词
|
||
result = getResolveResult(model1.getDescription());
|
||
String[] tmp = tags2.split(",");
|
||
count_tag = tmp.length;
|
||
tagList = changeArrayToList(tmp);
|
||
}else{
|
||
//表示项目1有标签
|
||
result = getResolveResult(model2.getDescription());
|
||
String[] tmp = tags1.split(",");
|
||
count_tag = tmp.length;
|
||
tagList = changeArrayToList(tmp);
|
||
}
|
||
if(stepOneResult){
|
||
//表示两个项目来自同一社区 匹配上的标签数量应该在50%以上
|
||
int threshold = 0;
|
||
if(count_tag % 2 == 0)
|
||
threshold = count_tag / 2;
|
||
else
|
||
threshold = count_tag / 2 + 1;
|
||
int count_match = matchListAndMap(tagList,result);
|
||
if(count_match >= threshold)
|
||
return true;
|
||
else
|
||
return false;
|
||
}else{
|
||
int threshold = 1;
|
||
int count_match = matchListAndMap(tagList, result);
|
||
if(count_match >= threshold)
|
||
return true;
|
||
else
|
||
return false;
|
||
}
|
||
}else{
|
||
//表示两者都没有标签
|
||
double similarity = calSimilarityByDescription(model1, model2);
|
||
if(stepOneResult){
|
||
//表示两个项目源自一个社区 去伪
|
||
//如果两个项目匹配结果 weight>0.9 表示是同一个项目
|
||
if(similarity > 0.9)
|
||
return true;
|
||
return false;
|
||
}else{
|
||
//表示两个项目不来自同一个社区 存真
|
||
//如果两个项目匹配结果 weight<0.1 表示不是同一个项目
|
||
if(similarity < 0.1)
|
||
return false;
|
||
return true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
//计算都有标签的项目之间的相似度
|
||
public double calSimilarityByTag(GatherProjectsModel model1, GatherProjectsModel model2){
|
||
// 读取item_tag_relation表中项目总数
|
||
// int wholeNum = dbSource.getItemNumber(taggingsTableName, "OpenSourceProject");
|
||
// 读取item_tag_relation表中该model对应的标签对象
|
||
|
||
// List<Taggings> taggingsList1 = dbSource.getTaggingsListByTaggableId(taggingsTableName, model1.getId());
|
||
// List<Taggings> taggingsList2 = dbSource.getTaggingsListByTaggableId(taggingsTableName, model2.getId());
|
||
//
|
||
// //读取每个name对应的tag对象
|
||
// List<Tags> tagList1 = getTagListByItemTagRelationList(taggingsList1);
|
||
// List<Tags> tagList2 = getTagListByItemTagRelationList(taggingsList2);
|
||
|
||
//获取权重向量
|
||
// Map<Integer, List<Double>> weightListMap = getWeightVectorByTag(tagList1, tagList2, wholeNum);
|
||
// List<Double> weightVector1 = weightListMap.get(1);
|
||
// List<Double> weightVector2 = weightListMap.get(2);
|
||
// return calDistance(weightVector1, weightVector2);
|
||
|
||
String[] tagArray_model1 = model1.getTags().split(",");
|
||
String[] tagArray_model2 = model2.getTags().split(",");
|
||
List<String> tags_model1 = changeArrayToList(tagArray_model1);
|
||
List<String> tags_model2 = changeArrayToList(tagArray_model2);
|
||
|
||
//比较两个list中相同标签的个数
|
||
List<String> sameTags = getSameStringsInList(tags_model1, tags_model2);
|
||
int minNum_tag = tags_model1.size() < tags_model2.size() ? tags_model1.size() : tags_model2.size();
|
||
float sameRate = (float)(sameTags.size() * 1.0 / minNum_tag);
|
||
System.out.println("same rate between teo tags : "+sameRate);
|
||
return sameRate;
|
||
}
|
||
|
||
//计算文本相似度
|
||
public double calSimilarityByDescription(GatherProjectsModel model1, GatherProjectsModel model2){
|
||
long time4_1 = System.currentTimeMillis();
|
||
String description1 = model1.getDescription();
|
||
String description2 = model2.getDescription();
|
||
Map<String, Integer> resolveResult1 = getResolveResult(description1);
|
||
Map<String, Integer> resolveResult2 = getResolveResult(description2);
|
||
Map<String, Double> tfResult1 = calTFOfDescription(resolveResult1);
|
||
Map<String, Double> tfResult2 = calTFOfDescription(resolveResult2);
|
||
Map<String, Double> idfResult = calIDFOfDescription(resolveResult1, resolveResult2);
|
||
|
||
List<String> commonList = new ArrayList<String>();
|
||
List<String> uniqueList1 = new ArrayList<String>();
|
||
List<String> uniqueList2 = new ArrayList<String>();
|
||
|
||
Set<String> keys1 = resolveResult1.keySet();
|
||
Iterator<String> it1 = keys1.iterator();
|
||
while(it1.hasNext()){
|
||
String key1 = it1.next();
|
||
if(resolveResult2.containsKey(key1)){
|
||
commonList.add(key1);
|
||
}else{
|
||
uniqueList1.add(key1);
|
||
}
|
||
}
|
||
|
||
Set<String> keys2 = resolveResult2.keySet();
|
||
Iterator<String> it2 = keys2.iterator();
|
||
while(it2.hasNext()){
|
||
String key1 = it2.next();
|
||
if(resolveResult1.containsKey(key1)){
|
||
//因为遍历第一个map时已经将commonList进行构造了 此处不用再add了
|
||
}else{
|
||
uniqueList2.add(key1);
|
||
}
|
||
}
|
||
|
||
List<Double> weightCommon1 = calWeightListOfDescription(commonList, tfResult1, idfResult);
|
||
List<Double> weightCommon2 = calWeightListOfDescription(commonList, tfResult2, idfResult);
|
||
List<Double> weightUnique1 = calWeightListOfDescription(uniqueList1, tfResult1, idfResult);
|
||
List<Double> weightUnique2 = calWeightListOfDescription(uniqueList2, tfResult2, idfResult);
|
||
|
||
List<Double> weightResult1 = new ArrayList<Double>();
|
||
List<Double> weightResult2 = new ArrayList<Double>();
|
||
|
||
for(int i = 0; i < weightCommon1.size(); i++){
|
||
weightResult1.add(weightCommon1.get(i));
|
||
weightResult2.add(weightCommon2.get(i));
|
||
}
|
||
|
||
for(int i = 0; i < weightUnique1.size(); i++){
|
||
weightResult1.add(weightUnique1.get(i));
|
||
weightResult2.add(0.0);
|
||
}
|
||
|
||
for(int i = 0; i < weightUnique2.size(); i++){
|
||
weightResult1.add(0.0);
|
||
weightResult2.add(weightUnique2.get(i));
|
||
}
|
||
long time4_2 = System.currentTimeMillis();
|
||
if(time4_2 - time4_1 > 10)
|
||
logger.info("time cost for cal similarity by description: " + (time4_2 - time4_1));
|
||
return calDistance(weightResult1, weightResult2);
|
||
}
|
||
|
||
|
||
|
||
//找两个向量组成的相似向量
|
||
public Map<Integer, List<Double>> getWeightVectorByTag(List<Tags> tagList1, List<Tags> tagList2, int wholeNum){
|
||
long time5_1 = System.currentTimeMillis();
|
||
List<Tags> commonTagList1 = new ArrayList<Tags>();//里面用Tag表示格式不同的相同标签 在取得时候两者都要取出来
|
||
List<Tags> commonTagList2 = new ArrayList<Tags>();
|
||
List<Tags> uniqueTag1 = new ArrayList<Tags>();
|
||
List<Tags> uniqueTag2 = new ArrayList<Tags>();
|
||
//首先要根据两边的tag组成向量
|
||
for(int i = 0; i < tagList1.size(); i++){
|
||
Tags tag1 = tagList1.get(i);
|
||
Tags similarTag;
|
||
if((similarTag = theMostSimilarTag(tag1, tagList2)) != null){
|
||
//表示存在相同的标签
|
||
commonTagList1.add(tag1);
|
||
commonTagList2.add(similarTag);
|
||
}
|
||
}
|
||
|
||
//找到各自独有的标签
|
||
uniqueTag1 = findUniqueTag(tagList1, commonTagList1);
|
||
uniqueTag2 = findUniqueTag(tagList2, commonTagList2);
|
||
|
||
List<Double> weightList1 = new ArrayList<Double>();
|
||
List<Double> weightList2 = new ArrayList<Double>();
|
||
|
||
for(int i = 0; i < commonTagList1.size(); i++){
|
||
Tags tag1 = commonTagList1.get(i);
|
||
Tags tag2 = commonTagList2.get(i);
|
||
double weight = calIDFOfTag(tag1, tag2, wholeNum);
|
||
weightList1.add(weight);
|
||
weightList2.add(weight);
|
||
}
|
||
|
||
for(int i = 0; i < uniqueTag1.size(); i++){
|
||
Tags tag1 = uniqueTag1.get(i);
|
||
double weight = calIDFOfTag(tag1, wholeNum);
|
||
weightList1.add(weight);
|
||
weightList2.add(0.0);
|
||
}
|
||
|
||
for(int i = 0; i < uniqueTag2.size(); i++){
|
||
Tags tag2 = uniqueTag2.get(i);
|
||
double weight = calIDFOfTag(tag2, wholeNum);
|
||
weightList2.add(weight);
|
||
weightList1.add(0.0);
|
||
}
|
||
|
||
Map<Integer, List<Double>> result = new HashMap<Integer, List<Double>>();
|
||
result.put(1, weightList1);
|
||
result.put(2, weightList2);
|
||
long time5_2 = System.currentTimeMillis();
|
||
if(time5_2 - time5_1 > 10)
|
||
logger.info("time cost for get weight vector by tag: " + (time5_2 - time5_1));
|
||
return result;
|
||
}
|
||
|
||
|
||
//找List各自独有的标签
|
||
public List<Tags> findUniqueTag(List<Tags> tagList, List<Tags> commonList){
|
||
List<Tags> result = new ArrayList<Tags>();
|
||
for(Tags tag:tagList){
|
||
if(!isExist(tag, commonList)){
|
||
//表示独有
|
||
result.add(tag);
|
||
}
|
||
}
|
||
return result;
|
||
}
|
||
|
||
//判断后者是否拥有前者
|
||
public boolean isExist(Tags tag, List<Tags> list){
|
||
for(Tags t:list){
|
||
if(tag.getId() == t.getId())
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
|
||
|
||
//找与该标签最相似的标签
|
||
public Tags theMostSimilarTag(Tags tag, List<Tags> tagList){
|
||
if(tag == null)
|
||
return null;
|
||
//对标签进行特殊字符去除处理
|
||
String tagName = RegexHandler.extractEngDecimalAndChinese(tag.getName());
|
||
for(Tags t:tagList){
|
||
if(t == null)
|
||
continue;
|
||
String tName = RegexHandler.extractEngDecimalAndChinese(t.getName());
|
||
if(tagName.equals(tName)){
|
||
//表示两个标签完全相同
|
||
return t;
|
||
}
|
||
}
|
||
return null;//表示没有找到相同的标签
|
||
}
|
||
|
||
|
||
public String theMostSimilarString(String str, List<String> strList){
|
||
if(str == null)
|
||
return null;
|
||
for(String s : strList){
|
||
if(s == null)
|
||
continue;
|
||
if(s.equals(str)){
|
||
//表示两个str相同
|
||
return str;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
|
||
//计算tag对应id的idf值
|
||
public double calIDFOfTag(Tags tag1, Tags tag2, int wholeNum){
|
||
double result = 0.00;
|
||
int count = dbSource.getNumOfTagIdAndOspByTwoTagIds(taggingsTableName, tag1.getId(), tag2.getId());
|
||
result = wholeNum * 1.0 / count;
|
||
result = Math.log(result);
|
||
return result;
|
||
}
|
||
//上面函数的重载
|
||
public double calIDFOfTag(Tags tag, int wholeNum){
|
||
double result = 0.00;
|
||
int count = dbSource.getNumOfTagIdAndOsp(taggingsTableName, tag.getId());
|
||
result = wholeNum * 1.0 / count + 0.0001;
|
||
result = Math.log(result);
|
||
return result;
|
||
}
|
||
|
||
//根据itemTagRelationList找到对应tag_id的Tag对象列表
|
||
public List<Tags> getTagListByItemTagRelationList(List<Taggings> itemTagRelationList){
|
||
List<Tags> result = new ArrayList<Tags>();
|
||
for(Taggings i:itemTagRelationList){
|
||
Tags t = dbSource.getTagById(tagsTableName, i.getTag_id());
|
||
result.add(t);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
|
||
//计算向量余弦夹角
|
||
public double calDistance(List<Double> vector1, List<Double> vector2){
|
||
double numerator = 0.0;//分子
|
||
double denominator = 0.0001;//分母
|
||
numerator = calNumerator(vector1, vector2);
|
||
denominator = calDenominator(vector1, vector2);
|
||
return numerator / denominator;
|
||
}
|
||
//余弦夹角分子计算
|
||
public double calNumerator(List<Double> vector1, List<Double> vector2){
|
||
Double result = 0.0;
|
||
for(int i = 0; i < vector1.size(); i++){
|
||
result += vector1.get(i) * vector2.get(i);
|
||
}
|
||
return result;
|
||
}
|
||
//余弦夹角分母计算
|
||
public double calDenominator(List<Double> vector1, List<Double> vector2){
|
||
Double result = 0.0001;
|
||
double result1 = 0.0;
|
||
for(Double d:vector1){
|
||
result1 += d * d;
|
||
}
|
||
result1 = Math.sqrt(result1);
|
||
|
||
double result2 = 0.0;
|
||
for(Double d:vector2){
|
||
result2 += d * d;
|
||
}
|
||
result2 = Math.sqrt(result2);
|
||
result = result1 * result2;
|
||
return result;
|
||
}
|
||
|
||
|
||
//对描述信息进行分词得到词频统计结果的Map
|
||
public Map<String, Integer> getResolveResult(String description){
|
||
long time6_1 = System.currentTimeMillis();
|
||
Map<String, Integer> result = new HashMap<String, Integer>();
|
||
Reader text = new StringReader(description);
|
||
IKSegmenter ik = new IKSegmenter(text, true);
|
||
Lexeme term = null;
|
||
try {
|
||
//处理一个分词结果
|
||
while((term = ik.next()) != null){
|
||
String tmp = term.getLexemeText().toString();
|
||
tmp = RegexHandler.extractEngDecimalAndChinese(tmp).toLowerCase();
|
||
if(result.containsKey(tmp))
|
||
result.put(tmp, result.get(tmp) + 1);
|
||
else
|
||
result.put(tmp, 1);
|
||
}
|
||
} catch (IOException e) {
|
||
// TODO Auto-generated catch block
|
||
e.printStackTrace();
|
||
}
|
||
long time6_2 = System.currentTimeMillis();
|
||
if(time6_2 - time6_1 > 10)
|
||
logger.info("time cost for get resolve result: " + (time6_2 - time6_1));
|
||
return result;
|
||
}
|
||
|
||
|
||
//计算文本匹配tf值
|
||
public Map<String, Double> calTFOfDescription(Map<String, Integer> map){
|
||
long time7_1 = System.currentTimeMillis();
|
||
Map<String, Double> result = new HashMap<String, Double>();
|
||
Set<String> keys = map.keySet();
|
||
Iterator<String> it = keys.iterator();
|
||
int tfAll = 0;
|
||
//计算当前描述所有词词频总和
|
||
while(it.hasNext()){
|
||
String name = it.next();
|
||
tfAll += map.get(name);
|
||
}
|
||
Iterator<String> it2 = keys.iterator();
|
||
while(it2.hasNext()){
|
||
String name = it2.next();
|
||
int tf = map.get(name);
|
||
double weight = tf * 1.0 / tfAll;
|
||
result.put(name, weight);
|
||
}
|
||
long time7_2 = System.currentTimeMillis();
|
||
if(time7_2 - time7_1 > 10)
|
||
logger.info("time cost for cal tf of description: " + (time7_2 - time7_1));
|
||
return result;
|
||
}
|
||
|
||
|
||
//计算文本匹配IDF值
|
||
public Map<String, Double> calIDFOfDescription(Map<String, Integer> map1, Map<String, Integer> map2){
|
||
long time8_1 = System.currentTimeMillis();
|
||
int docNum = 2;
|
||
|
||
Map<String, Double> result = new HashMap<String, Double>();
|
||
Map<String, Integer> count_n = new HashMap<String, Integer>();
|
||
Set<String> keys1 = map1.keySet();
|
||
Iterator<String> it1 = keys1.iterator();
|
||
while(it1.hasNext()){
|
||
String key = it1.next();
|
||
if(count_n.containsKey(key)){
|
||
count_n.put(key, count_n.get(key) + 1);
|
||
}else{
|
||
count_n.put(key, 1);
|
||
}
|
||
}
|
||
|
||
Set<String> keys2 = map2.keySet();
|
||
Iterator<String> it2 = keys2.iterator();
|
||
while(it2.hasNext()){
|
||
String key = it2.next();
|
||
if(count_n.containsKey(key)){
|
||
count_n.put(key, count_n.get(key) + 1);
|
||
}else{
|
||
count_n.put(key, 1);
|
||
}
|
||
}
|
||
|
||
Set<String> keys = count_n.keySet();
|
||
Iterator<String> it = keys.iterator();
|
||
while(it.hasNext()){
|
||
String name = it.next();
|
||
double weight = docNum / count_n.get(name) * 1.0 + 0.0001;
|
||
weight = Math.log(weight);
|
||
result.put(name, weight);
|
||
}
|
||
long time8_2 = System.currentTimeMillis();
|
||
if(time8_2 - time8_1 > 10)
|
||
logger.info("time cost for cal idf of description: " + (time8_2 - time8_1));
|
||
return result;
|
||
}
|
||
|
||
|
||
//根据文本的list得到weight的list
|
||
public List<Double> calWeightListOfDescription(List<String> textList, Map<String, Double> tfMap, Map<String, Double> idfMap){
|
||
long time9_1 = System.currentTimeMillis();
|
||
List<Double> result = new ArrayList<Double>();
|
||
for(int i = 0; i < textList.size(); i++){
|
||
double weight = tfMap.get(textList.get(i)) * idfMap.get(textList.get(i));
|
||
result.add(weight);
|
||
}
|
||
long time9_2 = System.currentTimeMillis();
|
||
if(time9_2 - time9_1 > 10)
|
||
logger.info("time cost for cal weight list of description: " + (time9_2 - time9_1));
|
||
return result;
|
||
}
|
||
|
||
|
||
|
||
//将linkNames存入数据库
|
||
public void insertSynonyms(List<String> linkNames){
|
||
for(String name:linkNames){
|
||
Synonyms model = new Synonyms();
|
||
model.setName(name.trim());
|
||
dbSource.insertSynonym(synonymTableName, model);
|
||
}
|
||
}
|
||
|
||
|
||
//插入数据到同义词表和项目关联表
|
||
public void insertSynonymming(List<String> linkNames, int synonymming_id){
|
||
for(String name:linkNames){
|
||
//查找同义词对象
|
||
Synonyms synonyms = dbSource.getSynonymByName(synonymTableName, name);
|
||
//如果synonyms漏掉 没有加入到数据库 则不执行后面的操作
|
||
if(synonyms == null)
|
||
continue;
|
||
Synonymmings synonymmings = new Synonymmings();
|
||
synonymmings.setSynonym_id(synonyms.getId());
|
||
synonymmings.setSynonymming_id(synonymming_id);
|
||
try {
|
||
dbSource.insertSynonymming(synonymmingTableName, synonymmings);
|
||
logger.info("project:" + synonymming_id + " has synonym:" + synonyms.getName());
|
||
} catch (Exception e) {
|
||
//在插入同义词时 gather_projects中的记录已经被删除掉了
|
||
logger.info("gather_projects_id has been removed");
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
//匹配完成后 将关联上的项目进行合并 结果存储到edd_relations表中 最后一个参数为true表示是更新表中的数据
|
||
public void handleAfterMatch(GatherProjectsModel model, List<String> synonyms, List<GatherProjectsModel> matchedProjectList, boolean updateOrNot){
|
||
insertSynonyms(synonyms);//插入新增项目的同义词
|
||
insertSynonymming(synonyms, model.getId());//插入新增项目和同义词之间的关联
|
||
|
||
//处理同义词表
|
||
for(int i = 0; i < matchedProjectList.size(); i++){
|
||
GatherProjectsModel model1 = matchedProjectList.get(i);
|
||
List<String> synonyms_1 = getSynonyms(model1);//获取同义词
|
||
//判断两个项目是不是来自同一个社区 如果是则不插入彼此的同义词
|
||
if(!((""+model.getSource()).equals(model1.getSource()+""))){
|
||
logger.info("project:" + model.getId() + " and project:" + model1.getId() + " add synonyms after match");
|
||
insertSynonymming(synonyms_1, model.getId());//在新增项目中插入关联项目同义词记录
|
||
insertSynonymming(synonyms, model1.getId());//在关联项目中插入新增项目的同义词记录
|
||
}
|
||
}
|
||
|
||
List<EddRelations> relationsList = new ArrayList<EddRelations>();
|
||
EddRelations relation_model = new EddRelations();
|
||
relation_model.setGather_projects_ids(","+model.getId()+",");
|
||
relationsList.add(relation_model); //relationsList用于记录包含当前项目在内的
|
||
|
||
List<EddRelations> relationsList_matched = new ArrayList<EddRelations>();//匹配上的项目在edd_relations表中的记录
|
||
for(int i = 0; i < matchedProjectList.size(); i++){
|
||
GatherProjectsModel model2 = matchedProjectList.get(i);
|
||
//查找edd_relations表得到相应的记录
|
||
List<EddRelations> eddRelationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model2.getId());
|
||
for(EddRelations relation:eddRelationList){
|
||
relationsList.add(relation);
|
||
relationsList_matched.add(relation);
|
||
}
|
||
if(eddRelationList.size() == 0){
|
||
//表示之前没有匹配上项目的匹配记录 需要插入这条记录 eg: ,12,
|
||
EddRelations relation = new EddRelations();
|
||
relation.setGather_projects_ids(","+model2.getId()+",");
|
||
relationsList.add(relation); //relationsList用于记录包含当前项目在内的
|
||
}
|
||
}
|
||
|
||
//relationsList_now表示目前正在处理的项目在edd_relations表中的记录 需要在插入新的edd_relations对象前查找
|
||
List<EddRelations> relationsList_now = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model.getId());
|
||
|
||
//统计relationList中所有不同的gather_projects_id
|
||
String relation_new_gather_projects_ids = hasDiffProjFromSameCommunity(relationsList);
|
||
EddRelations eddRelation = new EddRelations();
|
||
if(relation_new_gather_projects_ids != null){
|
||
//表示匹配结果中不存在同一个社区不同的多个项目匹配到一起
|
||
eddRelation.setGather_projects_ids(relation_new_gather_projects_ids);
|
||
dbSource.insertEddRelations(eddRelationTableName, eddRelation);//插入最新的结果集
|
||
/**
|
||
* 删除关联项目在edd_relations表中对应的记录
|
||
*/
|
||
for(EddRelations relation:relationsList_matched)
|
||
deleteExistEddRelations(relation);
|
||
/**
|
||
* 如果是更新项目 需要查看edd_relations表中是否有该记录 如果有要删除掉
|
||
*/
|
||
for(EddRelations relation:relationsList_now)
|
||
deleteExistEddRelations(relation);
|
||
}else{
|
||
//表示结果有来自同一社区的项目 如果当前项目在edd_relations表中找不到相应记录 则应插入本身的记录；反之不进行操作
|
||
if(relationsList_now.size() == 0){
|
||
//表示不是更新的项目 需要插入当前项目本身的记录
|
||
dbSource.insertEddRelations(eddRelationTableName, relation_model);
|
||
}
|
||
}
|
||
|
||
|
||
}
|
||
|
||
|
||
public int changeNullToInt(Integer i){
|
||
if(i == null)
|
||
return 0;
|
||
else
|
||
return i;
|
||
}
|
||
|
||
public List<String> changeArrayToList(String[] strs){
|
||
List<String> result = new ArrayList<String>();
|
||
for(String str:strs){
|
||
str = RegexHandler.extractEngDecimalAndChinese(str);//对标签进行处理
|
||
result.add(str);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
//匹配List中的数据和Map中的数据 返回匹配上得数量
|
||
public int matchListAndMap(List<String> tags, Map<String, Integer> description){
|
||
int count = 0;
|
||
for(String tag:tags){
|
||
if(description.containsKey(tag))
|
||
count++;
|
||
}
|
||
return count;
|
||
}
|
||
|
||
//找到当前项目同义词关联的项目 要求项目不相同 因为同一个项目可能包含多个同义词
|
||
public List<GatherProjectsModel> getRelatedGatherProjects(GatherProjectsModel model, List<String> synonymList){
|
||
List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
|
||
for(String name:synonymList){
|
||
Synonyms synonym = dbSource.getSynonymByName(synonymTableName, name);
|
||
if(synonym == null)
|
||
continue;//表示还不存在这样的标签
|
||
List<Synonymmings> synonymmingList = dbSource.selectSynonymmingListBySynonymId(synonymmingTableName, synonym.getId());
|
||
for(Synonymmings synonymming:synonymmingList){
|
||
GatherProjectsModel m = gatherDao.selectGPMById(gatherProjectsTableName, synonymming.getSynonymming_id());
|
||
if(model.getId() != m.getId()){
|
||
//判断m在result中是否存在
|
||
int i;
|
||
for(i = 0; i < result.size(); i++){
|
||
GatherProjectsModel tmp = result.get(i);
|
||
if(tmp.getId() == m.getId())
|
||
break;
|
||
}
|
||
if(i == result.size()){
|
||
result.add(m);
|
||
System.out.println("project:" + model.getId() + " match project:" + m.getId() + " by synonym:" + name);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return result;
|
||
}
|
||
|
||
//对url地址中的特殊字符进行转义处理
|
||
public String changeUrl(String url){
|
||
url = url.replace("/", "//");
|
||
url = url.replace("%", "/%");
|
||
url = url.replace("_", "/_");
|
||
url = url.replace("\\", "/\\");
|
||
return url;
|
||
}
|
||
|
||
|
||
//判断项目名的缩写是不是别名
|
||
public String getShortProjectName(String name, List<String> list){
|
||
String[] tmp = name.toLowerCase().split(" ");
|
||
String shortName = "";
|
||
for(String s:tmp){
|
||
String firstLetter = s.substring(0, 1);//取出第一个字符
|
||
shortName += firstLetter;
|
||
}
|
||
if(!"".equals(shortName)){
|
||
//如果该缩写在同义词结果中以单独的词存在 则说明是别名
|
||
for(String synonym:list){
|
||
//检查提取的别名中是否存在这个缩写形式
|
||
String[] words = synonym.split(" ");
|
||
for(String word:words){
|
||
if(shortName.equals(word)){
|
||
return shortName;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return "";//表示项目名的缩写不是项目的别名
|
||
}
|
||
|
||
//项目名是缩写 判断有没有相应的全称
|
||
public String getTotalProjectName(String name, List<String> list){
|
||
if(RegexHandler.othersExceptEng(name.toLowerCase())){
|
||
//表示项目名含有除小写英文字幕外的其他字符
|
||
return "";
|
||
}else{
|
||
//将项目名一个字符一个字符分隔开
|
||
String result = StringHandler.getFullName(name, list);
|
||
return result;//如果没有返回""
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 对一个新软件的全部操作
|
||
* @param args
|
||
*/
|
||
@Transactional(propagation=Propagation.REQUIRED)
|
||
public void handleNewProject(GatherProjectsModel model, boolean updateOrNot){
|
||
//提取项目的homepage
|
||
String homepage = model.getHomepage();
|
||
//如果是null 不需要查找相同homepage的项目
|
||
List<GatherProjectsModel> sameHomepageList = new ArrayList<GatherProjectsModel>();
|
||
if(homepage != null && !"".equals(homepage)){
|
||
homepage = homepage.trim();
|
||
//查找所有相同homepage字段的项目
|
||
//判断homepage是否有http://或https://
|
||
int index_http = homepage.indexOf("http://");
|
||
int index_https = homepage.indexOf("https://");
|
||
if(index_http == 0){
|
||
homepage = homepage.substring(index_http + 7); //去除http://
|
||
}else if(index_https == 0){
|
||
homepage = homepage.substring(index_https + 8); //去除https://
|
||
}else{
|
||
//表示不是以http或https开头
|
||
}
|
||
|
||
if((homepage.lastIndexOf("/") == homepage.length()-1) && (homepage.length() != 0)){
|
||
//如果homepage的最后一个字符是/ 需要去除掉
|
||
homepage = homepage.substring(0, homepage.length() - 1);
|
||
}
|
||
if(homepage.length() > 0){
|
||
//为homepage添加http或https头
|
||
String homepage1 = "http://" + homepage;
|
||
String homepage2 = "https://" + homepage;
|
||
String homepage3 = "http://" + homepage + "/";
|
||
String homepage4 = "https://" + homepage + "/";
|
||
|
||
sameHomepageList = gatherDao.selectGPMBySameHomePage(gatherProjectsTableName, homepage, homepage1, homepage2, homepage3, homepage4, model.getId(), model.getSource());//认为一定是相同的项目
|
||
|
||
//查看sameHomepageList和当前的model存不存在来自同一社区的项目 如果存在则认为homepage匹配有问题 就将sameHomepageList清空
|
||
|
||
|
||
for(GatherProjectsModel m : sameHomepageList){
|
||
logger.info("project " + model.getId() + " and " + m.getId() + " match by homepage");
|
||
}
|
||
}
|
||
}
|
||
|
||
//如果项目名由多个单词组成，并且相同 则认为两个项目相同!cvs-fast-export 连字符特殊情况
|
||
List<GatherProjectsModel> sameNameList = new ArrayList<GatherProjectsModel>();
|
||
if(StringHandler_ProjectName.getWordNum(model.getName()) > 1){
|
||
//表示项目名由多个单词组成
|
||
sameNameList = gatherDao.selectGPMBySameName(gatherProjectsTableName, model.getName(), model.getId()); //找到表中相同名称的项目
|
||
|
||
for(GatherProjectsModel m : sameNameList){
|
||
logger.info("project " + model.getId() + " and " + m.getId() + " match by name");
|
||
}
|
||
}
|
||
|
||
//提取项目的同义词
|
||
List<String> synonymList = getSynonyms(model);//获取项目的同义词
|
||
List<GatherProjectsModel> matchedProjectList = getRelatedGatherProjects(model, synonymList);//在之前已经处理过的项目中找到有相同同义词的项目列表
|
||
|
||
for(GatherProjectsModel m : matchedProjectList){
|
||
logger.info("project " + model.getId() + " and " + m.getId() + " match by synonyms");
|
||
}
|
||
|
||
for(int i = 0; i < matchedProjectList.size(); i++){
|
||
GatherProjectsModel model2 = matchedProjectList.get(i);
|
||
if(!areTwoProjectsTheSame(model, model2)){
|
||
matchedProjectList.remove(i);//对于不是同一项目的在匹配列表中去除掉
|
||
i--;
|
||
}
|
||
}
|
||
|
||
for(GatherProjectsModel m : matchedProjectList){
|
||
logger.info("project " + model.getId() + " and " + m.getId() + " match by synonyms and verify");
|
||
}
|
||
|
||
//需要将两个相同项目的集合中的项目进行合并
|
||
matchedProjectList = mergeTwoList(sameHomepageList, matchedProjectList);
|
||
matchedProjectList = mergeTwoList(sameNameList, matchedProjectList);
|
||
|
||
/**
|
||
* 对项目同义词表进行合并,并添加项目之间的关联(匹配完之后的所有操作)
|
||
* 是不是相同的项目都做相同的操作 因为在edd_relations表中不论是删除还是更新都是先删除再插入
|
||
*/
|
||
handleAfterMatch(model,synonymList,matchedProjectList, updateOrNot);
|
||
}
|
||
|
||
/**
|
||
* 判断edd_relations中的记录合并到一起会不会导致项目中有相同的项目匹配到一起
|
||
* 如果存在：返回null 如果不存在：返回匹配的结果
|
||
*/
|
||
//!!!!!!!!!同一社区直接删除
|
||
public String hasDiffProjFromSameCommunity(List<EddRelations> list){
|
||
String result = ",";
|
||
Map<Integer, Integer> map_project = new HashMap<Integer, Integer>();
|
||
Map<String, Integer> map_source = new HashMap<String, Integer>();
|
||
for(EddRelations relation:list){
|
||
//统计不同的项目id
|
||
String gather_projects_ids = relation.getGather_projects_ids();
|
||
gather_projects_ids = gather_projects_ids.substring(1, gather_projects_ids.length() - 1);
|
||
String[] idArray = gather_projects_ids.split(",");
|
||
for(String id:idArray)
|
||
map_project.put(Integer.parseInt(id), 1);
|
||
}
|
||
Set<Integer> idSet = map_project.keySet();
|
||
Iterator it = idSet.iterator();
|
||
while(it.hasNext()){
|
||
int id = (int)it.next();
|
||
GatherProjectsModel model = gatherDao.selectGPMById(gatherProjectsTableName, id);
|
||
String source = model.getSource();
|
||
if(map_source.containsKey(source)){
|
||
return null;//表示有重复社区的项目
|
||
}else{
|
||
map_source.put(source, 1);
|
||
result += id + ",";
|
||
}
|
||
}
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 删除edd_relations表中对应id的记录 如果有osp_id则也要删除对应的open_source_projects表中的记录
|
||
*/
|
||
public void deleteExistEddRelations(EddRelations relation){
|
||
//删除之前对应的relation记录
|
||
int edd_relation_id = relation.getId();
|
||
dbSource.deleteEddRelationsItem(eddRelationTableName, edd_relation_id);
|
||
}
|
||
|
||
/**
|
||
* 将两个list<GatherProjectsModel>进行合并
|
||
*/
|
||
public List<GatherProjectsModel> mergeTwoList(List<GatherProjectsModel> list1, List<GatherProjectsModel> list2){
|
||
List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
|
||
Map<GatherProjectsModel, Integer> map = new HashMap<GatherProjectsModel, Integer>();
|
||
for(GatherProjectsModel model:list1){
|
||
if(!map.containsKey(model))
|
||
map.put(model, 1);
|
||
}
|
||
for(GatherProjectsModel model:list2){
|
||
if(!map.containsKey(model))
|
||
map.put(model, 1);
|
||
}
|
||
Set<GatherProjectsModel> keySet = map.keySet();
|
||
Iterator it = keySet.iterator();
|
||
while(it.hasNext()){
|
||
result.add((GatherProjectsModel) it.next());
|
||
}
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 比较两个list中相同个数
|
||
*/
|
||
public List<String> getSameStringsInList(List<String> list1, List<String> list2){
|
||
List<String> result = new ArrayList<String>();
|
||
for(int i = 0; i < list1.size(); i++){
|
||
String str = list1.get(i);
|
||
String sameString = theMostSimilarString(str, list2);
|
||
if(sameString != null)
|
||
result.add(sameString);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
|
||
/**
|
||
* 判断是不是有来自同一社区的项目
|
||
*/
|
||
|
||
}
|