modify some erros of GetSynonms
This commit is contained in:
parent
c81f788a09
commit
8dbf6f62ac
|
|
@ -48,7 +48,7 @@ public class GetSynonyms{
|
|||
@Resource
|
||||
private PlatformProjectDao paltformDao;
|
||||
|
||||
private static String gatherProjectsTableName =TableName.gatherProjectsTableName;
|
||||
private static String gatherProjectsTableName ="gather_projects_tmp";//TableName.gatherProjectsTableName;
|
||||
private static String synonymsTableName = TableName.synonymsTableName;
|
||||
private static String pointerTableName = TableName.pointerTableName;
|
||||
private static int batchSize = 1000;
|
||||
|
|
@ -112,158 +112,198 @@ public class GetSynonyms{
|
|||
}
|
||||
public Map<String,Integer> synonymsFilter(Map<String,Integer> synonymMap,String targetTable,GatherProjectsModel model)
|
||||
{
|
||||
Map<String,Integer> copy = synonymMap;
|
||||
for(String key:copy.keySet())
|
||||
{
|
||||
if(copy.get(key) == 1)
|
||||
{
|
||||
GatherProjectsModel tmp = model;
|
||||
//通过别名查找别名表中具有相同别名且flag为1的项目Id
|
||||
Set<Integer> dupSynonymPrj =getSameSynonymPrj(key,targetTable);
|
||||
|
||||
//GitHub社区项目中存在A-B-C，A B C这种，需要把短横、下划线去掉后再次判别是否是相同别名
|
||||
String replSpecial = ReplaceSpecial(key);
|
||||
if(!replSpecial.equals(key))
|
||||
try
|
||||
{
|
||||
Map<String,Integer> copy = synonymMap;
|
||||
if(copy != null)
|
||||
for(String key:copy.keySet())
|
||||
{
|
||||
if(copy.get(key) == 1)
|
||||
{
|
||||
Set<Integer> dupSynonymsPrj2 = getSameSynonymPrj(replSpecial, targetTable);
|
||||
if(dupSynonymsPrj2 != null)
|
||||
dupSynonymPrj.addAll(dupSynonymsPrj2);
|
||||
}
|
||||
|
||||
if(dupSynonymPrj != null)
|
||||
{
|
||||
Iterator<Integer> iterator = dupSynonymPrj.iterator();
|
||||
while(iterator.hasNext())
|
||||
GatherProjectsModel tmp = model;
|
||||
//通过别名查找别名表中具有相同别名且flag为1的项目Id
|
||||
Set<Integer> dupSynonymPrj =getSameSynonymPrj(key,targetTable);
|
||||
|
||||
//GitHub社区项目中存在A-B-C，A B C这种，需要把短横、下划线去掉后再次判别是否相同。
|
||||
String replSpecial = ReplaceSpecial(key);
|
||||
if(!replSpecial.equals(key))
|
||||
{
|
||||
GatherProjectsModel prj = dbSource.getGatherProjectById(gatherProjectsTableName ,iterator.next());
|
||||
//迭代求取最优项目，保证tmp始终代表最适合这一别名的项目
|
||||
tmp = getBetterPrj(tmp,prj);
|
||||
Set<Integer> dupSynonymsPrj2 = getSameSynonymPrj(replSpecial, targetTable);
|
||||
if(dupSynonymsPrj2 != null)
|
||||
{
|
||||
if(dupSynonymPrj != null && dupSynonymsPrj2 != null)
|
||||
dupSynonymPrj.addAll(dupSynonymsPrj2);
|
||||
else
|
||||
dupSynonymPrj = dupSynonymsPrj2;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(dupSynonymPrj != null)
|
||||
{
|
||||
Iterator<Integer> iterator = dupSynonymPrj.iterator();
|
||||
while(iterator.hasNext())
|
||||
{
|
||||
GatherProjectsModel prj = dbSource.getGatherProjectById(gatherProjectsTableName ,iterator.next());
|
||||
//迭代求取最优项目，保证tmp始终代表最适合这一别名的项目
|
||||
if(prj != null)
|
||||
tmp = getBetterPrj(tmp,prj);
|
||||
}
|
||||
}
|
||||
|
||||
if(tmp.getId()==model.getId() && tmp != null)
|
||||
{
|
||||
//当前项目为最适合这一别名的项目，就更新别名表中其余项目的flag为0
|
||||
dbDest.updateSynoymmings(targetTable,key,model.getId(),0);
|
||||
}
|
||||
else
|
||||
{
|
||||
synonymMap.put(key, 0);
|
||||
dbDest.updateSynoymmings(targetTable, key, tmp.getId(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
if(tmp.getId()==model.getId() && tmp != null)
|
||||
{
|
||||
//当前项目为最适合这一别名的项目，就更新别名表中其余项目的flag为0
|
||||
dbDest.updateSynoymmings(targetTable,key,model.getId(),0);
|
||||
}
|
||||
else
|
||||
{
|
||||
synonymMap.put(key, 0);
|
||||
dbDest.updateSynoymmings(targetTable, key, tmp.getId(), 0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
return synonymMap;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
return synonymMap;
|
||||
|
||||
}
|
||||
|
||||
public Set<Integer> getSameSynonymPrj(String key,String targetTable)
|
||||
{
|
||||
Set<Integer> dupSynonymPrj = new HashSet<Integer>();
|
||||
dupSynonymPrj = dbDest.selectSameSynonymPrj(targetTable,key,1);
|
||||
|
||||
return dupSynonymPrj;
|
||||
try
|
||||
{
|
||||
Set<Integer> dupSynonymPrj = new HashSet<Integer>();
|
||||
dupSynonymPrj = dbDest.selectSameSynonymPrj(targetTable,key,1);
|
||||
|
||||
return dupSynonymPrj;
|
||||
}catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 比较两个项目哪一个更适合某一别名
|
||||
* */
|
||||
private GatherProjectsModel getBetterPrj(GatherProjectsModel model1, GatherProjectsModel model2) {
|
||||
// TODO Auto-generated method stub
|
||||
GatherProjectsModel betterPrj=null;
|
||||
//不同的社区赋给不同的值
|
||||
int communityValue1 = getCommunityValue(model1);
|
||||
int communityValue2 = getCommunityValue(model2);
|
||||
//如果来自同一社区就从抽取数据库中查询其有代表意义的数据比较热度
|
||||
if(communityValue1 == communityValue2)
|
||||
try
|
||||
{
|
||||
if(communityValue1 == -1)
|
||||
return model1;
|
||||
else
|
||||
|
||||
// TODO Auto-generated method stub
|
||||
GatherProjectsModel betterPrj=null;
|
||||
//不同的社区赋给不同的值
|
||||
int communityValue1 = getCommunityValue(model1);
|
||||
int communityValue2 = getCommunityValue(model2);
|
||||
//如果来自同一社区就从抽取数据库中查询其有代表意义的数据比较热度
|
||||
if(communityValue1 == communityValue2)
|
||||
{
|
||||
if(communityValue1 == 4)
|
||||
if(communityValue1 == -1)
|
||||
return model1;
|
||||
else
|
||||
{
|
||||
/**oschina用star_num具有代表性，
|
||||
**用List是因为抽取数据库中未经汇总，有相同项目存在，只需求出list取第一个值即可
|
||||
**/
|
||||
List<Integer> hot1 = paltformDao.getHotValue("oschina_project","star_num",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("oschina_project","star_num",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1.size() == 0) return model2;
|
||||
else return model1;
|
||||
|
||||
}
|
||||
if(communityValue1 == 3)
|
||||
{
|
||||
List<Integer> hot1 = paltformDao.getHotValue("openhub_project","user_num",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("openhub_project","user_num",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1.size() == 0) return model2;
|
||||
else return model1;
|
||||
}
|
||||
if(communityValue1 == 2)
|
||||
{
|
||||
List<Integer> hot1 = paltformDao.getHotValue("sourceforge_project","download_num",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("sourceforge_project","download_num",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1.size() == 0) return model2;
|
||||
else return model1;
|
||||
}
|
||||
if(communityValue1 == 1)
|
||||
{
|
||||
List<Integer> hot1 = paltformDao.getHotValue("github","forks",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("github","forks",model2.getUrl_md5());
|
||||
if(hot1 != null && hot2 != null)
|
||||
return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1== null)
|
||||
return model2;
|
||||
if(communityValue1 == 4)
|
||||
{
|
||||
/**oschina用star_num具有代表性，
|
||||
**用List是因为抽取数据库中未经汇总，有相同项目存在，只需求出list取第一个值即可
|
||||
**/
|
||||
List<Integer> hot1 = paltformDao.getHotValue("oschina_project","star_num",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("oschina_project","star_num",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
return model1;
|
||||
}
|
||||
|
||||
if(communityValue1 == 0)
|
||||
{
|
||||
//freecode项目抽取结果中只有创建时间可以利用，创建时间较早的认为比较可信
|
||||
DateFormat df1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
DateFormat df2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
List<String> time1 = paltformDao.getTime("gather_projects", "extracted_time", model1.getUrl_md5());
|
||||
List<String> time2 = paltformDao.getTime("gather_projects", "extracted_time",model2.getUrl_md5());
|
||||
if(time1!=null && time2!=null)
|
||||
try {
|
||||
Date dt1 = df1.parse(time1.get(0));
|
||||
Date dt2 = df2.parse(time2.get(0));
|
||||
if(dt1 !=null && dt2!=null)
|
||||
{
|
||||
if(dt1.getTime() > dt2.getTime())
|
||||
return model1;
|
||||
else
|
||||
return model2;
|
||||
}
|
||||
} catch (ParseException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
else
|
||||
if(time1 != null)
|
||||
return model1;
|
||||
else
|
||||
return model2;
|
||||
if(hot1.size() == 0)
|
||||
return model2;
|
||||
else
|
||||
return model1;
|
||||
|
||||
}
|
||||
if(communityValue1 == 3)
|
||||
{
|
||||
List<Integer> hot1 = paltformDao.getHotValue("openhub_project","user_num",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("openhub_project","user_num",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1.size() == 0)
|
||||
return model2;
|
||||
else
|
||||
return model1;
|
||||
}
|
||||
if(communityValue1 == 2)
|
||||
{
|
||||
List<Integer> hot1 = paltformDao.getHotValue("sourceforge_project","download_num",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("sourceforge_project","download_num",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)
|
||||
return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1.size() == 0)
|
||||
return model2;
|
||||
else
|
||||
return model1;
|
||||
}
|
||||
if(communityValue1 == 1)
|
||||
{
|
||||
List<Integer> hot1 = paltformDao.getHotValue("github_tmp","forks",model1.getUrl_md5());
|
||||
List<Integer> hot2 = paltformDao.getHotValue("github_tmp","forks",model2.getUrl_md5());
|
||||
if(hot1.size()>0 && hot2.size()>0)
|
||||
return hot1.get(0) > hot2.get(0) ? model1:model2;
|
||||
else
|
||||
if(hot1== null || hot1.size()==0)
|
||||
return model2;
|
||||
else
|
||||
return model1;
|
||||
}
|
||||
|
||||
if(communityValue1 == 0)
|
||||
{
|
||||
//freecode项目抽取结果中只有创建时间可以利用，创建时间较早的认为比较可信
|
||||
DateFormat df1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
DateFormat df2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
List<String> time1 = paltformDao.getTime("gather_projects", "extracted_time", model1.getUrl_md5());
|
||||
List<String> time2 = paltformDao.getTime("gather_projects", "extracted_time",model2.getUrl_md5());
|
||||
if(time1!=null && time2!=null)
|
||||
try {
|
||||
Date dt1 = df1.parse(time1.get(0));
|
||||
Date dt2 = df2.parse(time2.get(0));
|
||||
if(dt1 !=null && dt2!=null)
|
||||
{
|
||||
if(dt1.getTime() > dt2.getTime())
|
||||
return model1;
|
||||
else
|
||||
return model2;
|
||||
}
|
||||
} catch (ParseException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
else
|
||||
if(time1 != null)
|
||||
return model1;
|
||||
else
|
||||
return model2;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
return communityValue1 > communityValue2 ? model1:model2;
|
||||
|
||||
return betterPrj;
|
||||
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
else
|
||||
return communityValue1 > communityValue2 ? model1:model2;
|
||||
|
||||
return betterPrj;
|
||||
}
|
||||
/**
|
||||
* 根据项目的不同社区，赋给其不同的社区热度值
|
||||
|
|
@ -334,7 +374,6 @@ public class GetSynonyms{
|
|||
String subName = getSubName(prjName);
|
||||
if(subName != null)
|
||||
{
|
||||
System.out.println("从名字中提取到：" + subName);
|
||||
SynonymMap.put(subName, 1);
|
||||
}
|
||||
|
||||
|
|
@ -423,7 +462,6 @@ public class GetSynonyms{
|
|||
if(!"".equals(projectNameWithoutComName))
|
||||
{
|
||||
synonymsList.add(projectNameWithoutComName);
|
||||
logger.info("gets synonyms:" + projectNameWithoutComName + " Without ComName and remove racket");
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -613,12 +651,11 @@ public class GetSynonyms{
|
|||
|
||||
for(String name:linkNameResult)
|
||||
{
|
||||
logger.info("be 动词之前的有： "+name);
|
||||
if(!keywords.contains(name.toLowerCase().trim()) && !name.toLowerCase().trim().equals(subName.toLowerCase().trim()) )
|
||||
{
|
||||
if(DeleteSpecial(name).equals(strDelete) && DeleteSpecial(name)!= null)
|
||||
{
|
||||
System.out.println("1位置提取到别名：" + name);
|
||||
|
||||
synonymsList.add(name);
|
||||
}
|
||||
else
|
||||
|
|
@ -628,7 +665,6 @@ public class GetSynonyms{
|
|||
|
||||
if(synonymsFromGitPrj != null)
|
||||
{
|
||||
System.out.println("2位置提取到：" + synonymsFromGitPrj);
|
||||
synonymsList.add(synonymsFromGitPrj);
|
||||
}
|
||||
else
|
||||
|
|
@ -671,7 +707,6 @@ public class GetSynonyms{
|
|||
public void handleLinkNameResult(GatherProjectsModel model,List<String> linkNameResult,List<String> synonymsList,List<String>linkNameList,List<String>keywords)
|
||||
{
|
||||
for(String name:linkNameResult){
|
||||
logger.info("be 动词之前的有： "+name);
|
||||
//be动词前不包括项目类型，且和项目名不相同
|
||||
if(!keywords.contains(name.toLowerCase().trim()) && !name.toLowerCase().trim().equals(model.getName().toLowerCase().trim())
|
||||
&& !name.toLowerCase().trim().equals(projectNameWithoutComName.toLowerCase().trim())){
|
||||
|
|
@ -731,7 +766,6 @@ public class GetSynonyms{
|
|||
//去除短横线
|
||||
if(subName.charAt(i) == '-')
|
||||
{
|
||||
//System.out.println("字符位置在" + i + " and 字符是" + subName.charAt(i));
|
||||
str.deleteCharAt(i-deleteNum);
|
||||
deleteNum++;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
<property name="url"
|
||||
value="jdbc:mysql://localhost/ossean_production?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="1234" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
|
|
@ -40,7 +40,7 @@
|
|||
<property name="url"
|
||||
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="1234" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
|
|
|
|||
Loading…
Reference in New Issue