gyiang
/
test_ll
0
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: gyiang:master
Branches Tags
gyiang:master
gyiang:ddk
gyiang:ff1
gyiang:ff2
..
compare: gyiang:ff1
Branches Tags
gyiang:ddk
gyiang:ff1
gyiang:ff2
gyiang:master

No commits in common. "master" and "ff1" have entirely different histories.
master ... ff1

12 changed files with 1 additions and 263 deletions
Show Stats Expand all files Collapse all files

7
.classpath
View File

@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
<classpathentry kind="lib" path="D:/Projects/Java/SE/DailyScheduler/lib/sigar.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>

1
.gitignore vendored
View File

@ -1 +0,0 @@
/.idea

17
.project
View File

@ -1,17 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>DailyScheduler</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

11
.settings/org.eclipse.jdt.core.prefs
View File

@ -1,11 +0,0 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.source=1.7

27
DailyScheduler.iml
View File

@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="EclipseModuleManager">
<libelement value="file://D:/Projects/Java/SE/DailyScheduler/lib/sigar.jar" />
<src_description expected_position="0">
<src_folder value="file://$MODULE_DIR$/src" expected_position="0" />
</src_description>
</component>
<component name="NewModuleRootManager" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/bin" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="jdk" jdkName="JavaSE-1.7" jdkType="JavaSDK" />
<orderEntry type="module-library">
<library name="sigar.jar">
<CLASSES>
<root url="file://D:/Projects/Java/SE/DailyScheduler/lib/sigar.jar" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
</component>
</module>

1
R.txt Normal file
View File

@ -0,0 +1 @@
我是一个好人，这是一张一张哈哈哈哈

BIN
bin/com/ossean/crawler/dailyScheduler/DailyScheduler.class
View File

Binary file not shown.

BIN
bin/com/ossean/crawler/dailyScheduler/MyTask.class
View File

Binary file not shown.

BIN
lib/sigar.jar
View File

Binary file not shown.

181
src/com/ossean/crawler/dailyScheduler/DailyScheduler.java
View File

@ -1,181 +0,0 @@
package com.ossean.crawler.dailyScheduler;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.LinkedList;
import java.util.Queue;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.hyperic.sigar.Mem;
import org.hyperic.sigar.Sigar;
import org.hyperic.sigar.SigarException;
class MyTask implements Runnable{
private SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");//设置日期格式
String cmdPrefix = "sh bin/";
String cmd;
long rotateInterval = 1000 * 60 * 60 * 1;// schedule some tasks every 2 hours for memory limit
int memoPerTask = 850;//the memory a task needs
Runtime rt;
Process process;
BufferedReader bufferedReader;
Queue<String> tasks = new LinkedList<String>();
//String cmd = cmdPrefix + "bytes.sh";
public void schedule(){
long lastSchedule = System.currentTimeMillis();//上一次的调度时间
long now;//此次调度结束的时间
long sleepTime;//距离下次调度要睡眠的时间
while(true){
// step 1: read tasks to be scheduled today
readTasks();
// step 2: do job
doSchedule();
now = System.currentTimeMillis();
sleepTime = (24 * 60 * 60 * 1000) - (now - lastSchedule);
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
lastSchedule = System.currentTimeMillis();
}
}
private void doSchedule() {
while(true){
System.out.println("check out the free memory...");
// step 3.1: whether enough memory can be used
long freeMemo = 0;
try {
Sigar sigar = new Sigar();
Mem mem = sigar.getMem();
freeMemo = mem.getFree() / (1024 * 1024);
System.out.println("free memo: " + freeMemo + " @ " + df.format(new Date()));
} catch (SigarException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
if(freeMemo < (memoPerTask + 600)){
System.out.println("not enough memo");
try {
Thread.sleep(60 * 20 * 1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
continue;
}
//step 3.2: schedule some task
int scheduleNum = (int)((freeMemo - 600) / memoPerTask);
while(scheduleNum-- > 0 && tasks.size() > 0){
String task = tasks.poll().trim();
System.out.println(">>>>>>>> schedule " + task + " @ " + df.format(new Date()));
//kill the task scheduled yesterday if exists
killProcess(task);
this.cmd = this.cmdPrefix + task+".sh";
rt = Runtime.getRuntime();
try {
process = rt.exec(cmd,new String[]{},new File("/home/pdl/programs/dailyScheduledCrawler/fetch_networks/"));
bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()), 1024);
process.waitFor();
/* String line;
System.out.println("the output of script");
while((line = bufferedReader.readLine()) != null){
System.out.println(line);
}*/
System.out.println(">>>>>>>> schedule done " + df.format(new Date()));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
if(tasks.size() == 0){
System.out.println("today's schedule is over "+ df.format(new Date()));
break;
}
try {
System.out.println("sleep for rotatenterval time");
Thread.sleep(this.rotateInterval);
System.out.println("sleep over");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
private void killProcess(String task) {
try {
rt = Runtime.getRuntime();
process = rt.exec("ps -aux|grep " + task + " |grep -v grep|cut -c 9-15|xargs kill -9" );
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void readTasks() {
String fileName = "tasks.txt";
File file = new File(fileName);
BufferedReader reader = null;
try {
System.out.println("begain to read task list from file tasks.txt" + " @ " + df.format(new Date()));
reader = new BufferedReader(new FileReader(file));
String tempString = null;
System.out.println("tasks are : >>>>>");
while ((tempString = reader.readLine()) != null) {
System.out.println(tempString);
tasks.add(tempString);
}
reader.close();
System.out.println("done to read task " + " @ " + df.format(new Date()));
} catch (IOException e) {
e.printStackTrace();
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
}
}
}
}
@Override
public void run() {
schedule();
}
}
public class DailyScheduler {
static ScheduledThreadPoolExecutor stpe = null;
public static void main(String[] args) {
//stpe = new ScheduledThreadPoolExecutor(2);
MyTask task = new MyTask();
System.out.println("here we go");
task.schedule();
//stpe.scheduleAtFixedRate(task, 0, 24*60*60, TimeUnit.SECONDS);
}
}

17
tasks.txt
View File

@ -1,17 +0,0 @@
csdn_ask
slashdot
iteye_blog
cnblogs_q_unsolved
neitui
ibm_post
code_project
cnblogs_news
oschina_question
51cto_blog
csdn_topic
softpedia
bytes
oschina_project
sourceforge
openhub
csdn_blog

2
test-push.txt
View File

@ -1,2 +0,0 @@
lalalalalalalalal
akvanbjasnbkn