From bfb0a035fcd88d48eac2c0d023242d4495449b49 Mon Sep 17 00:00:00 2001 From: ossean Date: Mon, 14 Nov 2016 17:21:05 +0800 Subject: [PATCH] complete dail_scheduler --- .gitignore | 1 + crawler/daily_scheduler/delete.sh | 2 +- .../dailyScheduler/DailyScheduler.java | 208 ++++++++++++++++++ crawler/daily_scheduler/tasks.txt | 11 + crawler/daily_scheduler/tasks.txt.backup | 19 ++ 5 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java create mode 100644 crawler/daily_scheduler/tasks.txt create mode 100644 crawler/daily_scheduler/tasks.txt.backup diff --git a/.gitignore b/.gitignore index 027071ba2..a2ee3643a 100644 --- a/.gitignore +++ b/.gitignore @@ -114,5 +114,6 @@ /crawler/dailyScheduledCrawler/fetch_networks/target/ /crawler/dailyScheduledCrawler/fetch_networks/log/* /crawler/daily_scheduler/log/* +/crawler/daily_scheduler/log_mem/* /crawler/moreSmarterCrawler/fetch_networks/target/ /crawler/moreSmarterCrawler/fetch_networks/log/* diff --git a/crawler/daily_scheduler/delete.sh b/crawler/daily_scheduler/delete.sh index a52b0c4d7..d45d2ad99 100644 --- a/crawler/daily_scheduler/delete.sh +++ b/crawler/daily_scheduler/delete.sh @@ -1 +1 @@ -ps -aux|grep oschina_project |grep -v grep|cut -c 9-15|xargs kill -9 \ No newline at end of file +ps -aux|grep oschina_question |grep -v grep|cut -c 9-15|xargs kill -9 \ No newline at end of file diff --git a/crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java b/crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java new file mode 100644 index 000000000..0a328785c --- /dev/null +++ b/crawler/daily_scheduler/src/com/ossean/crawler/dailyScheduler/DailyScheduler.java @@ -0,0 +1,208 @@ +package com.ossean.crawler.dailyScheduler; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.LinkedList; +import java.util.Queue; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.hyperic.sigar.Mem; +import org.hyperic.sigar.Sigar; +import org.hyperic.sigar.SigarException; + +class MyTask implements Runnable{ + private SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式 + String cmdPrefix = "sh bin/"; + String cmd; + long rotateInterval = 1000 * 60 * 60 * 1;// schedule some tasks every 1 hours for memory limit + int memoPerTask = 800;//the memory a task needs + int memoLeft = 600; + Runtime rt; + Process process; + BufferedReader bufferedReader; + Queue tasks = new LinkedList(); + //String cmd = cmdPrefix + "bytes.sh"; + + public void schedule(){ + long lastSchedule = System.currentTimeMillis();//上一次的调度时间 + long now;//此次调度结束的时间 + long sleepTime;//距离下次调度要睡眠的时间 + + while(true){ + System.out.println("----------------------------------------------------- "); + System.out.println(">>>>>schedule start @ " + df.format(new Date())); + // step 1: read tasks to be scheduled today + readTasks(); + + // step 2: do job + doSchedule(); + System.out.println(">>>>>schedule done @ " + df.format(new Date())); + now = System.currentTimeMillis(); + long period = (now - lastSchedule); + if(period < 24 * 60 * 60 * 1000){ + sleepTime = (24 * 60 * 60 * 1000) - period; + try { + System.out.println(">>>>>sleep " + sleepTime /1000 + "s @ " + df.format(new Date())); + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + System.out.println("----------------------------------------------------- "); + lastSchedule = System.currentTimeMillis(); + + } + } + private void doSchedule() { + + while(true){ + //System.out.println("check out the free memory..."); + // step 3.1: whether enough memory can be used + long freeMemo = 0; + try { + Sigar sigar = new Sigar(); + Mem mem = sigar.getMem(); + freeMemo = mem.getFree() / (1024 * 1024); + + //System.out.println("free memo: " + freeMemo + " @ " + df.format(new Date())); + } catch (SigarException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + if(freeMemo < (memoPerTask + memoLeft )){ + //System.out.println("not enough memo"); + try { + Thread.sleep(60 * 20 * 1000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + continue; + } + //step 3.2: schedule some task + int scheduleNum = (int)((freeMemo - memoLeft ) / memoPerTask); + while(scheduleNum-- > 0 && tasks.size() > 0){ + String task = tasks.poll().trim(); + System.out.print( task + " @ " + df.format(new Date()) + " ,"); + //kill the task scheduled yesterday if exists + killProcess(task); + this.cmd = this.cmdPrefix + task+".sh"; + rt = Runtime.getRuntime(); + try { + process = rt.exec(cmd,new String[]{},new File("/home/pdl/programs/dailyScheduledCrawler/fetch_networks/")); + bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024); + process.waitFor(); + String line; + // System.out.println("the output of script"); + // while((line = bufferedReader.readLine()) != null){ + // System.out.println(line); + // } + // System.out.println(">>>>>>>> schedule done " + df.format(new Date())); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + if(tasks.size() == 0){ + //System.out.println("today's schedule is over "+ df.format(new Date())); + break; + } + + try { + //System.out.println("sleep for rotatenterval time"); + System.out.println( ); + Thread.sleep(this.rotateInterval); + //System.out.println("sleep over"); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + private void killProcess(String task) { + + try{ + //System.out.println(" kill process:" +task); + rt = Runtime.getRuntime(); + String deleteStr = "ps -aux|grep " + task + " |grep -v grep|cut -c 9-15|xargs kill -9" ; + FileWriter fileWritter = new FileWriter(new File("delete.sh").getName(),false); + BufferedWriter bufferWritter = new BufferedWriter(fileWritter); + bufferWritter.write(deleteStr); + bufferWritter.close(); + + process = rt.exec("sh delete.sh"); + bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024); + process.waitFor(); + String line; + // System.out.println("the output of kill:"); + // while((line = bufferedReader.readLine()) != null){ + // System.out.println(line); + //} + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + private void readTasks() { + String fileName = "tasks.txt"; + File file = new File(fileName); + BufferedReader reader = null; + try { + //System.out.println("begain to read task list from file tasks.txt" + " @ " + df.format(new Date())); + reader = new BufferedReader(new FileReader(file)); + String tempString = null; + // System.out.println("tasks are : >>>>>"); + while ((tempString = reader.readLine()) != null) { + // System.out.println(tempString); + tasks.add(tempString); + } + reader.close(); + // System.out.println("done to read task " + " @ " + df.format(new Date())); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e1) { + } + } + } + } + @Override + public void run() { + schedule(); + } +} +public class DailyScheduler { + static ScheduledThreadPoolExecutor stpe = null; + public static void main(String[] args) { + + //stpe = new ScheduledThreadPoolExecutor(2); + MyTask task = new MyTask(); + // System.out.println("here we go"); + task.schedule(); + //stpe.scheduleAtFixedRate(task, 0, 24*60*60, TimeUnit.SECONDS); + + } +} diff --git a/crawler/daily_scheduler/tasks.txt b/crawler/daily_scheduler/tasks.txt new file mode 100644 index 000000000..1c25e9058 --- /dev/null +++ b/crawler/daily_scheduler/tasks.txt @@ -0,0 +1,11 @@ +csdn_ask +iteye_blog +cnblogs_q_unsolved +neitui +code_project +cnblogs_news +oschina_question +51cto_blog +csdn_topic +dewen_question +cnblogs_q_solved diff --git a/crawler/daily_scheduler/tasks.txt.backup b/crawler/daily_scheduler/tasks.txt.backup new file mode 100644 index 000000000..298ddfd19 --- /dev/null +++ b/crawler/daily_scheduler/tasks.txt.backup @@ -0,0 +1,19 @@ +csdn_ask +slashdot +iteye_blog +cnblogs_q_unsolved +neitui +ibm_post +code_project +cnblogs_news +oschina_question +51cto_blog +csdn_topic +softpedia +bytes +oschina_project +sourceforge +openhub +csdn_blog +dewen_question +cnblogs_q_solved \ No newline at end of file