117 lines
4.1 KiB
Java
117 lines
4.1 KiB
Java
package com.ossean;
|
|
|
|
import java.text.DateFormat;
|
|
import java.text.ParseException;
|
|
import java.text.SimpleDateFormat;
|
|
import java.util.Date;
|
|
import java.util.HashSet;
|
|
import java.util.List;
|
|
import java.util.Set;
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
import org.apache.log4j.Logger;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Qualifier;
|
|
import org.springframework.context.ApplicationContext;
|
|
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import com.ossean.dao.DBSource;
|
|
import com.ossean.dao.GatherDao;
|
|
import com.ossean.dao.UpdateControlProjectsDao;
|
|
import com.ossean.model.GatherProjectsModel;
|
|
import com.ossean.util.MergeProjectNew2;
|
|
|
|
@Component
|
|
public class MergeProjects {
|
|
Logger logger = Logger.getLogger(this.getClass());
|
|
@Resource
|
|
private DBSource dbSource;
|
|
@Resource
|
|
private GatherDao gatherDao;
|
|
@Resource
|
|
private UpdateControlProjectsDao updateControlDao;
|
|
@Qualifier("mergeProjectNew2")
|
|
@Autowired
|
|
private MergeProjectNew2 mergeProjectNew2;
|
|
private static Date extractedTime;
|
|
|
|
private static String pointerTableName = TableName.pointerTableName;//记录去重进行到哪个地方了
|
|
private static String sourceTableName = TableName.gatherProjectsTableName;//去重项目的来源，来自汇总程序生成的表
|
|
private static String targetTableName = TableName.eddRelationTableName;//去重结果存储位置
|
|
private static int batchSize = 500;
|
|
//读指针
|
|
public int readPointer(String table, String source, String target, int minId){
|
|
int pointer = minId;
|
|
try {
|
|
pointer = dbSource.getPointer(table, source, target);
|
|
} catch(Exception e) {
|
|
logger.info("No such pointer! Create one");
|
|
dbSource.insertPointer(table, source, target, pointer);
|
|
}
|
|
return pointer;
|
|
}
|
|
|
|
public void start(){
|
|
logger.info("start remove projects!");
|
|
int count=0;
|
|
int tmpCount = 0;
|
|
count = readPointer(pointerTableName,sourceTableName,targetTableName, count);//指针表计量处理的项目数
|
|
long start = System.currentTimeMillis();
|
|
String timeStr = gatherDao.selectMaxExtractedTime(sourceTableName);
|
|
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
|
try {
|
|
extractedTime = dateFormat.parse(timeStr);
|
|
} catch (ParseException e1) {
|
|
e1.printStackTrace();
|
|
}
|
|
logger.info("current max extractedTime: "+extractedTime.toString());
|
|
while(true){
|
|
//取完别名
|
|
List<GatherProjectsModel> gpmList = gatherDao.selectGPMListNew(sourceTableName,1,batchSize);
|
|
if(gpmList.size()==0){
|
|
logger.info("Remove Duplicate Projects:all projects have been done. Sleep 10 mins");
|
|
logger.warn("deal with "+tmpCount+" projects cost: "+(float)(System.currentTimeMillis() - start)/600000+" minutes");
|
|
try {
|
|
Thread.sleep(600*1000);
|
|
} catch (InterruptedException e) {
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
else{
|
|
for(GatherProjectsModel model:gpmList){
|
|
int handleCount = 0;
|
|
logger.info("Duplicate remove:handling project " + model.getId());
|
|
String modelTimeStr = model.getExtracted_time();
|
|
Date modelExtractedTime = null;
|
|
try {
|
|
modelExtractedTime = dateFormat.parse(modelTimeStr);
|
|
if(modelExtractedTime.after(extractedTime))
|
|
//扫全表
|
|
handleCount = mergeProjectNew2.handleNewProject(model,true);
|
|
else
|
|
handleCount = mergeProjectNew2.handleNewProject(model,false);
|
|
} catch (ParseException e) {
|
|
e.printStackTrace();
|
|
}
|
|
count = count + handleCount;
|
|
tmpCount = tmpCount + handleCount;
|
|
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, count);
|
|
}
|
|
|
|
if(count%10000==0)
|
|
logger.warn("deal with:"+count+" projects cost: "+(float)(System.currentTimeMillis() - start)/60000+" minutes");
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
|
|
public static void main(String[] args){
|
|
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
|
MergeProjects Main = applicationContext.getBean(MergeProjects.class);
|
|
Main.start();
|
|
}
|
|
}
|