complete dail_scheduler
This commit is contained in:
ossean
parent
fc7e91d163
commit
bfb0a035fc
|
|
@ -114,5 +114,6 @@
|
|||
/crawler/dailyScheduledCrawler/fetch_networks/target/
|
||||
/crawler/dailyScheduledCrawler/fetch_networks/log/*
|
||||
/crawler/daily_scheduler/log/*
|
||||
/crawler/daily_scheduler/log_mem/*
|
||||
/crawler/moreSmarterCrawler/fetch_networks/target/
|
||||
/crawler/moreSmarterCrawler/fetch_networks/log/*
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
ps -aux|grep oschina_project |grep -v grep|cut -c 9-15|xargs kill -9
|
||||
ps -aux|grep oschina_question |grep -v grep|cut -c 9-15|xargs kill -9
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package com.ossean.crawler.dailyScheduler;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Queue;
|
||||
import java.util.Timer;
|
||||
import java.util.TimerTask;
|
||||
import java.util.concurrent.ScheduledThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.hyperic.sigar.Mem;
|
||||
import org.hyperic.sigar.Sigar;
|
||||
import org.hyperic.sigar.SigarException;
|
||||
|
||||
class MyTask implements Runnable{
|
||||
private SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式
|
||||
String cmdPrefix = "sh bin/";
|
||||
String cmd;
|
||||
long rotateInterval = 1000 * 60 * 60 * 1;// schedule some tasks every 1 hours for memory limit
|
||||
int memoPerTask = 800;//the memory a task needs
|
||||
int memoLeft = 600;
|
||||
Runtime rt;
|
||||
Process process;
|
||||
BufferedReader bufferedReader;
|
||||
Queue<String> tasks = new LinkedList<String>();
|
||||
//String cmd = cmdPrefix + "bytes.sh";
|
||||
|
||||
public void schedule(){
|
||||
long lastSchedule = System.currentTimeMillis();//上一次的调度时间
|
||||
long now;//此次调度结束的时间
|
||||
long sleepTime;//距离下次调度要睡眠的时间
|
||||
|
||||
while(true){
|
||||
System.out.println("----------------------------------------------------- ");
|
||||
System.out.println(">>>>>schedule start @ " + df.format(new Date()));
|
||||
// step 1: read tasks to be scheduled today
|
||||
readTasks();
|
||||
|
||||
// step 2: do job
|
||||
doSchedule();
|
||||
System.out.println(">>>>>schedule done @ " + df.format(new Date()));
|
||||
now = System.currentTimeMillis();
|
||||
long period = (now - lastSchedule);
|
||||
if(period < 24 * 60 * 60 * 1000){
|
||||
sleepTime = (24 * 60 * 60 * 1000) - period;
|
||||
try {
|
||||
System.out.println(">>>>>sleep " + sleepTime /1000 + "s @ " + df.format(new Date()));
|
||||
Thread.sleep(sleepTime);
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println("----------------------------------------------------- ");
|
||||
lastSchedule = System.currentTimeMillis();
|
||||
|
||||
}
|
||||
}
|
||||
private void doSchedule() {
|
||||
|
||||
while(true){
|
||||
//System.out.println("check out the free memory...");
|
||||
// step 3.1: whether enough memory can be used
|
||||
long freeMemo = 0;
|
||||
try {
|
||||
Sigar sigar = new Sigar();
|
||||
Mem mem = sigar.getMem();
|
||||
freeMemo = mem.getFree() / (1024 * 1024);
|
||||
|
||||
//System.out.println("free memo: " + freeMemo + " @ " + df.format(new Date()));
|
||||
} catch (SigarException e1) {
|
||||
// TODO Auto-generated catch block
|
||||
e1.printStackTrace();
|
||||
}
|
||||
if(freeMemo < (memoPerTask + memoLeft )){
|
||||
//System.out.println("not enough memo");
|
||||
try {
|
||||
Thread.sleep(60 * 20 * 1000);
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
//step 3.2: schedule some task
|
||||
int scheduleNum = (int)((freeMemo - memoLeft ) / memoPerTask);
|
||||
while(scheduleNum-- > 0 && tasks.size() > 0){
|
||||
String task = tasks.poll().trim();
|
||||
System.out.print( task + " @ " + df.format(new Date()) + " ,");
|
||||
//kill the task scheduled yesterday if exists
|
||||
killProcess(task);
|
||||
this.cmd = this.cmdPrefix + task+".sh";
|
||||
rt = Runtime.getRuntime();
|
||||
try {
|
||||
process = rt.exec(cmd,new String[]{},new File("/home/pdl/programs/dailyScheduledCrawler/fetch_networks/"));
|
||||
bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024);
|
||||
process.waitFor();
|
||||
String line;
|
||||
// System.out.println("the output of script");
|
||||
// while((line = bufferedReader.readLine()) != null){
|
||||
// System.out.println(line);
|
||||
// }
|
||||
// System.out.println(">>>>>>>> schedule done " + df.format(new Date()));
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(tasks.size() == 0){
|
||||
//System.out.println("today's schedule is over "+ df.format(new Date()));
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
//System.out.println("sleep for rotatenterval time");
|
||||
System.out.println( );
|
||||
Thread.sleep(this.rotateInterval);
|
||||
//System.out.println("sleep over");
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
private void killProcess(String task) {
|
||||
|
||||
try{
|
||||
//System.out.println(" kill process:" +task);
|
||||
rt = Runtime.getRuntime();
|
||||
String deleteStr = "ps -aux|grep " + task + " |grep -v grep|cut -c 9-15|xargs kill -9" ;
|
||||
FileWriter fileWritter = new FileWriter(new File("delete.sh").getName(),false);
|
||||
BufferedWriter bufferWritter = new BufferedWriter(fileWritter);
|
||||
bufferWritter.write(deleteStr);
|
||||
bufferWritter.close();
|
||||
|
||||
process = rt.exec("sh delete.sh");
|
||||
bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024);
|
||||
process.waitFor();
|
||||
String line;
|
||||
// System.out.println("the output of kill:");
|
||||
// while((line = bufferedReader.readLine()) != null){
|
||||
// System.out.println(line);
|
||||
//}
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
private void readTasks() {
|
||||
String fileName = "tasks.txt";
|
||||
File file = new File(fileName);
|
||||
BufferedReader reader = null;
|
||||
try {
|
||||
//System.out.println("begain to read task list from file tasks.txt" + " @ " + df.format(new Date()));
|
||||
reader = new BufferedReader(new FileReader(file));
|
||||
String tempString = null;
|
||||
// System.out.println("tasks are : >>>>>");
|
||||
while ((tempString = reader.readLine()) != null) {
|
||||
// System.out.println(tempString);
|
||||
tasks.add(tempString);
|
||||
}
|
||||
reader.close();
|
||||
// System.out.println("done to read task " + " @ " + df.format(new Date()));
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if (reader != null) {
|
||||
try {
|
||||
reader.close();
|
||||
} catch (IOException e1) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public void run() {
|
||||
schedule();
|
||||
}
|
||||
}
|
||||
public class DailyScheduler {
|
||||
static ScheduledThreadPoolExecutor stpe = null;
|
||||
public static void main(String[] args) {
|
||||
|
||||
//stpe = new ScheduledThreadPoolExecutor(2);
|
||||
MyTask task = new MyTask();
|
||||
// System.out.println("here we go");
|
||||
task.schedule();
|
||||
//stpe.scheduleAtFixedRate(task, 0, 24*60*60, TimeUnit.SECONDS);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
csdn_ask
|
||||
iteye_blog
|
||||
cnblogs_q_unsolved
|
||||
neitui
|
||||
code_project
|
||||
cnblogs_news
|
||||
oschina_question
|
||||
51cto_blog
|
||||
csdn_topic
|
||||
dewen_question
|
||||
cnblogs_q_solved
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
csdn_ask
|
||||
slashdot
|
||||
iteye_blog
|
||||
cnblogs_q_unsolved
|
||||
neitui
|
||||
ibm_post
|
||||
code_project
|
||||
cnblogs_news
|
||||
oschina_question
|
||||
51cto_blog
|
||||
csdn_topic
|
||||
softpedia
|
||||
bytes
|
||||
oschina_project
|
||||
sourceforge
|
||||
openhub
|
||||
csdn_blog
|
||||
dewen_question
|
||||
cnblogs_q_solved
|
||||
Loading…
Reference in New Issue