natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1904 from lfyee/dev 

Remove ScheduleUtil，use the existing CronUtils
This commit is contained in:
老佛爷 2020-02-07 21:35:24 +08:00 committed by GitHub
parent fccbcc8d7d 0eec3b8661
commit 4a25127f6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 64 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dolphinscheduler-api/pom.xml
View File

@ -43,10 +43,6 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-dao</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-server</artifactId>
</dependency>
<!--springboot-->
<dependency>

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java
View File

@ -30,7 +30,7 @@ import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.server.utils.ScheduleUtils;
import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -512,8 +512,7 @@ public class ExecutorService extends BaseService{
List<Date> listDate = new LinkedList<>();
if(!CollectionUtils.isEmpty(schedules)){
for (Schedule item : schedules) {
List<Date> list = ScheduleUtils.getRecentTriggerTime(item.getCrontab(), start, end);
listDate.addAll(list);
listDate.addAll(CronUtils.getSelfFireDateList(start, end, item.getCrontab()));
}
}
if(!CollectionUtils.isEmpty(listDate)){

2
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecutorService2Test.java
View File

@ -203,7 +203,7 @@ public class ExecutorService2Test {
"", "", RunMode.RUN_MODE_PARALLEL,
Priority.LOW, 0, 110);
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
verify(processDao, times(16)).createCommand(any(Command.class));
verify(processDao, times(15)).createCommand(any(Command.class));
}catch (Exception e){
Assert.assertTrue(false);
}

17
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/cron/CronUtils.java
View File

@ -156,6 +156,23 @@ public class CronUtils {
return dateList;
}
/**
* gets all scheduled times for a period of time based on self dependency
* @param startTime startTime
* @param endTime endTime
* @param cron cron
* @return date list
*/
public static List<Date> getSelfFireDateList(Date startTime, Date endTime, String cron) {
CronExpression cronExpression = null;
try {
cronExpression = parse2CronExpression(cron);
}catch (ParseException e){
logger.error(e.getMessage(), e);
return Collections.EMPTY_LIST;
}
return getSelfFireDateList(startTime, endTime, cronExpression);
}
/**
* get expiration time

65
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/cron/CronUtilsTest.java
View File

@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.dao.cron;
import org.apache.dolphinscheduler.common.enums.CycleEnum;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
import com.cronutils.builder.CronBuilder;
import com.cronutils.model.Cron;
@ -31,10 +32,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.ParseException;
import java.util.Date;
import static com.cronutils.model.field.expression.FieldExpressionFactory.*;
/**
* CronUtilsTest
*/
public class CronUtilsTest {
@ -55,8 +58,9 @@ public class CronUtilsTest {
.withSecond(on(0))
.instance();
// Obtain the string expression
String cronAsString = cron.asString(); // 0 */5 * * * ? * Every five minutes(once every 5 minutes)
String cronAsString = cron.asString();
// 0 */5 * * * ? * Every five minutes(once every 5 minutes)
Assert.assertEquals(cronAsString, "0 */5 * * * ? *");
}
@ -68,9 +72,6 @@ public class CronUtilsTest {
@Test
public void testCronParse() throws ParseException {
String strCrontab = "0 1 2 3 * ? *";
strCrontab = "0/50 0/59 * * * ? *";
strCrontab = "3/5 * 0/5 * * ? *";
strCrontab = "1/5 3/5 1/5 3/30 * ? *";
Cron depCron = CronUtils.parse2Cron(strCrontab);
Assert.assertEquals(depCron.retrieve(CronFieldName.SECOND).getExpression().asString(), "0");
@ -87,12 +88,14 @@ public class CronUtilsTest {
*/
@Test
public void testScheduleType() throws ParseException {
CycleEnum cycleEnum = CronUtils.getMaxCycle("0 */1 * * * ? *");
CycleEnum cycleEnum = CronUtils.getMaxCycle(CronUtils.parse2Cron("0 */1 * * * ? *"));
Assert.assertEquals(cycleEnum.name(), "MINUTE");
CycleEnum cycleEnum2 = CronUtils.getMaxCycle("0 * * * * ? *");
Assert.assertEquals(cycleEnum2.name(), "MINUTE");
CycleEnum cycleEnum3 = CronUtils.getMiniCycle(CronUtils.parse2Cron("0 * * * * ? *"));
Assert.assertEquals(cycleEnum3.name(), "MINUTE");
}
/**
@ -109,26 +112,9 @@ public class CronUtilsTest {
.withMinute(every(5))
.withSecond(on(0))
.instance();
String cronAsString = cron1.asString(); // 0 */5 * * * ? * once every 5 minutes
//logger.info(cronAsString);
// Obtain the string expression
//String minCrontab = "0 0 * * * ? *";
//String minCrontab = "0 0 10,14,16 * * ?";
//String minCrontab = "0 0-5 14 * * ? *";
//String minCrontab = "0 0 2 ? * SUN *";
//String minCrontab = "* 0,3 2 SUN * 1#1 *";
//String minCrontab = "* 0,3 * 1W * ? *";
//cron = CronUtils.parse2Cron("0 * * * * ? *");
// month cycle
/*String[] cronArayy = new String[]{"* 0,3 * 1W * ? *","* 0 0 1W * ? *",
"0 0 0 L 3/5 ? *","0 0 0 ? 3/5 2/2 *"};*/
// minute cycle
String[] cronArayy = new String[]{"* * * * * ? *","* 0 * * * ? *",
"* 5 * * 3/5 ? *","0 0 * * * ? *"};
// week cycle
/*String[] cronArayy = new String[]{"* * * ? * 2/1 *","0 *//*5 * ? * 2/1 *",
"* * *//*5 ? * 2/1 *"};*/
for(String minCrontab:cronArayy){
if (!org.quartz.CronExpression.isValidExpression(minCrontab)) {
throw new RuntimeException(minCrontab+" verify failure, cron expression not valid");
@ -171,7 +157,6 @@ public class CronUtilsTest {
logger.info("dayOfWeekField instanceof And:"+(dayOfWeekField.getExpression() instanceof And));
logger.info("dayOfWeekField instanceof QuestionMark:"+(dayOfWeekField.getExpression() instanceof QuestionMark));
CycleEnum cycleEnum = CronUtils.getMaxCycle(minCrontab);
if(cycleEnum !=null){
logger.info(cycleEnum.name());
@ -180,4 +165,34 @@ public class CronUtilsTest {
}
}
}
}
@Test
public void getSelfFireDateList() throws ParseException{
Date from = DateUtils.stringToDate("2020-01-01 00:00:00");
Date to = DateUtils.stringToDate("2020-01-31 00:00:00");
// test date
Assert.assertEquals(0, CronUtils.getSelfFireDateList(to, from, "0 0 0 * * ? ").size());
// test error cron
Assert.assertEquals(0, CronUtils.getSelfFireDateList(from, to, "0 0 0 * *").size());
// test cron
Assert.assertEquals(29, CronUtils.getSelfFireDateList(from, to, "0 0 0 * * ? ").size());
// test other
Assert.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? ")).size());
Assert.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2CronExpression("0 0 0 * * ? "), 5).size());
}
@Test
public void getExpirationTime(){
Date startTime = DateUtils.stringToDate("2020-02-07 18:30:00");
Date expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.HOUR);
Assert.assertEquals("2020-02-07 19:30:00", DateUtils.dateToString(expirationTime));
expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.DAY);
Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime));
expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.WEEK);
Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime));
expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.MONTH);
Assert.assertEquals("2020-02-07 23:59:59", DateUtils.dateToString(expirationTime));
expirationTime = CronUtils.getExpirationTime(startTime, CycleEnum.YEAR);
Assert.assertEquals("2020-02-07 18:30:00", DateUtils.dateToString(expirationTime));
}
}

5
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java
View File

@ -33,9 +33,9 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.dao.utils.cron.CronUtils;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.AlertManager;
import org.apache.dolphinscheduler.server.utils.ScheduleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -213,8 +213,7 @@ public class MasterExecThread implements Runnable {
List<Date> listDate = Lists.newLinkedList();
if(!CollectionUtils.isEmpty(schedules)){
for (Schedule schedule : schedules) {
List<Date> list = ScheduleUtils.getRecentTriggerTime(schedule.getCrontab(), startDate, endDate);
listDate.addAll(list);
listDate.addAll(CronUtils.getSelfFireDateList(startDate, endDate, schedule.getCrontab()));
}
}
// get first fire date

79
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ScheduleUtils.java
View File

@ -1,79 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.quartz.impl.triggers.CronTriggerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.ParseException;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
/**
* ScheduleUtils
*/
public class ScheduleUtils {
private static final Logger logger = LoggerFactory.getLogger(ScheduleUtils.class);
/**
* Get the execution time of the time interval
* @param cron
* @param from
* @param to
* @return
*/
public static List<Date> getRecentTriggerTime(String cron, Date from, Date to) {
return getRecentTriggerTime(cron, Integer.MAX_VALUE, from, to);
}
/**
* Get the execution time of the time interval
* @param cron
* @param size
* @param from
* @param to
* @return
*/
public static List<Date> getRecentTriggerTime(String cron, int size, Date from, Date to) {
List list = new LinkedList<Date>();
if(to.before(from)){
logger.error("schedule date from:{} must before date to:{}!", from, to);
return list;
}
try {
CronTriggerImpl trigger = new CronTriggerImpl();
trigger.setCronExpression(cron);
trigger.setStartTime(from);
trigger.setEndTime(to);
trigger.computeFirstFireTime(null);
for (int i = 0; i < size; i++) {
Date schedule = trigger.getNextFireTime();
if(null == schedule){
break;
}
list.add(schedule);
trigger.triggered(null);
}
} catch (ParseException e) {
logger.error("cron:{} error:{}", cron, e.getMessage());
}
return java.util.Collections.unmodifiableList(list);
}
}

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/MasterExecThreadTest.java
View File

@ -134,7 +134,7 @@ public class MasterExecThreadTest {
method.setAccessible(true);
method.invoke(masterExecThread);
// one create save, and 15(1 to 31 step 2) for next save, and last day 31 no save
verify(processDao, times(16)).saveProcessInstance(processInstance);
verify(processDao, times(15)).saveProcessInstance(processInstance);
}catch (Exception e){
Assert.assertTrue(false);
}

44
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ScheduleUtilsTest.java
View File

@ -1,44 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.junit.Test;
import java.util.Date;
import static org.junit.Assert.assertEquals;
/**
* Test ScheduleUtils
*/
public class ScheduleUtilsTest {
/**
* Test the getRecentTriggerTime method
*/
@Test
public void testGetRecentTriggerTime() {
Date from = DateUtils.stringToDate("2020-01-01 00:00:00");
Date to = DateUtils.stringToDate("2020-01-31 01:00:00");
// test date
assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", to, from).size());
// test error cron
assertEquals(0, ScheduleUtils.getRecentTriggerTime("0 0 0 * *", from, to).size());
// test cron
assertEquals(31, ScheduleUtils.getRecentTriggerTime("0 0 0 * * ? ", from, to).size());
}
}

2
pom.xml
View File

@ -706,12 +706,12 @@
<include>**/server/utils/SparkArgsUtilsTest.java</include>
<include>**/server/utils/FlinkArgsUtilsTest.java</include>
<include>**/server/utils/ParamUtilsTest.java</include>
<include>**/server/utils/ScheduleUtilsTest.java</include>
<include>**/server/master/MasterExecThreadTest.java</include>
<include>**/dao/mapper/AccessTokenMapperTest.java</include>
<include>**/dao/mapper/AlertGroupMapperTest.java</include>
<include>**/dao/mapper/AlertMapperTest.java</include>
<include>**/dao/mapper/CommandMapperTest.java</include>
<include>**/dao/cron/CronUtilsTest.java</include>
<include>**/alert/template/AlertTemplateFactoryTest.java</include>
<include>**/alert/template/impl/DefaultHTMLTemplateTest.java</include>
<include>**/server/worker/task/datax/DataxTaskTest.java</include>