109 lines
3.6 KiB
Java
109 lines
3.6 KiB
Java
package com.ossean;
|
|
|
|
import java.util.List;
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
import org.apache.log4j.Logger;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Qualifier;
|
|
import org.springframework.context.ApplicationContext;
|
|
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import com.ossean.databaseSource.DBSource;
|
|
import com.ossean.databaseSource.GatherDao;
|
|
import com.ossean.databaseSource.UpdateControlProjectsDao;
|
|
import com.ossean.model.GatherProjectsModel;
|
|
import com.ossean.model.UpdateControlProjects;
|
|
import com.ossean.util.MergeProjectsUtil;
|
|
|
|
@Component
|
|
public class MergeProjects {
|
|
Logger logger = Logger.getLogger(this.getClass());
|
|
@Resource
|
|
private DBSource dbSource;
|
|
@Resource
|
|
private GatherDao gatherDao;
|
|
@Resource
|
|
private UpdateControlProjectsDao updateControlDao;
|
|
@Qualifier("mergeProjectsUtil")
|
|
@Autowired
|
|
private MergeProjectsUtil mergeProjectsUtil;
|
|
|
|
private int startId1;
|
|
|
|
private static String pointerTableName = "edd_pointers";
|
|
private static String sourceTableName = "gather_projects";
|
|
private static String targetTableName = "edd_relations";
|
|
private static String updateControlProjectsTableName = "update_control_projects";
|
|
|
|
private static int batchSize = 500;
|
|
|
|
//读指针
|
|
public int readPointer(String table, String source, String target){
|
|
int pointer = 1;
|
|
//pointer的初始值应该从数据库中读出最小的id
|
|
try {
|
|
pointer = dbSource.getMinId(source);
|
|
} catch (Exception e){
|
|
//表示表中没有数据
|
|
logger.info("No item in this table");
|
|
dbSource.insertPointer(table, source, target, 1);
|
|
return 1;
|
|
}
|
|
|
|
try {
|
|
pointer = dbSource.getPointer(table, source, target);
|
|
} catch(Exception e) {
|
|
logger.info("No such pointer! Create one");
|
|
dbSource.insertPointer(table, source, target, pointer);
|
|
}
|
|
return pointer;
|
|
}
|
|
|
|
public void start(){
|
|
while(true){
|
|
//查看更新表中有没有需要更新的数据
|
|
List<UpdateControlProjects> updateProjectList = updateControlDao.findAllItems(updateControlProjectsTableName);
|
|
if(updateProjectList.size() != 0){
|
|
//表示有需要更新的项目
|
|
for(UpdateControlProjects updateProject:updateProjectList){
|
|
//重新处理更新项目
|
|
GatherProjectsModel model = gatherDao.selectGPMById(sourceTableName, updateProject.getId());
|
|
mergeProjectsUtil.handleNewProject(model, true);
|
|
updateControlDao.deleteOneItem(updateControlProjectsTableName, updateProject.getId());
|
|
}
|
|
}
|
|
|
|
startId1 = readPointer(pointerTableName, sourceTableName, targetTableName);
|
|
List<GatherProjectsModel> gpmList1 = gatherDao.selectGPMList(sourceTableName, startId1, batchSize);
|
|
if(gpmList1.size() == 0){
|
|
try {
|
|
logger.info("no item to be handled! Sleep 1h");
|
|
Thread.sleep(3600*1000);
|
|
continue;
|
|
} catch (InterruptedException e) {
|
|
// TODO Auto-generated catch block
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
for(GatherProjectsModel model1:gpmList1){
|
|
logger.info("handling project : " + model1.getId());
|
|
// List<GatherProjectsModel> sameProjects = gatherDao.selectGPMByUrlMD5(sourceTableName, model1.getHomepageMD5());
|
|
/**
|
|
* 内部函数中省略了相同homepage_MD5的代码片段
|
|
*/
|
|
mergeProjectsUtil.handleNewProject(model1, false);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
public static void main(String[] args){
|
|
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
|
MergeProjects Main = applicationContext.getBean(MergeProjects.class);
|
|
Main.start();
|
|
}
|
|
}
|