natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Improvement][Service] Optimize query log (#5205) 

* [Improvement][Service] Optimize query log

* remove IPUtils
This commit is contained in:
wenjun 2021-04-15 20:25:01 +08:00 committed by GitHub
parent b6453da298
commit f3dfaacca2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 12 additions and 274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java
View File

@ -1,63 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
/**
* http utils
*/
public class IpUtils {
private IpUtils() {
throw new UnsupportedOperationException("Construct IpUtils");
}
public static final String DOT = ".";
/**
* ip str to long <p>
*
* @param ipStr ip string
* @return ip to long
*/
public static Long ipToLong(String ipStr) {
String[] ipSet = ipStr.split("\\" + DOT);
return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]);
}
/**
* long to ip
*
* @param ipLong the long number converted from IP
* @return String
*/
public static String longToIp(long ipLong) {
long[] ipNumbers = new long[4];
long tmp = 0xFF;
ipNumbers[0] = ipLong >> 24 & tmp;
ipNumbers[1] = ipLong >> 16 & tmp;
ipNumbers[2] = ipLong >> 8 & tmp;
ipNumbers[3] = ipLong & tmp;
return ipNumbers[0] + DOT
+ ipNumbers[1] + DOT
+ ipNumbers[2] + DOT
+ ipNumbers[3];
}
}

55
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.junit.Assert;
import org.junit.Test;
public class IpUtilsTest {
@Test
public void ipToLong() {
String ip = "192.168.110.1";
String ip2 = "0.0.0.0";
long longNumber = IpUtils.ipToLong(ip);
long longNumber2 = IpUtils.ipToLong(ip2);
System.out.println(longNumber);
Assert.assertEquals(3232263681L, longNumber);
Assert.assertEquals(0L, longNumber2);
String ip3 = "255.255.255.255";
long longNumber3 = IpUtils.ipToLong(ip3);
System.out.println(longNumber3);
Assert.assertEquals(4294967295L, longNumber3);
}
@Test
public void longToIp() {
String ip = "192.168.110.1";
String ip2 = "0.0.0.0";
long longNum = 3232263681L;
String i1 = IpUtils.longToIp(longNum);
String i2 = IpUtils.longToIp(0);
Assert.assertEquals(ip, i1);
Assert.assertEquals(ip2, i2);
}
}

4
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java
View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.remote.utils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@ -48,7 +50,7 @@ public class Constants {
public static final int CPUS = Runtime.getRuntime().availableProcessors();
public static final String LOCAL_ADDRESS = IPUtils.getFirstNoLoopbackIP4Address();
public static final String LOCAL_ADDRESS = NetUtils.getHost();
/**
* netty epoll enable switch

145
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/IPUtils.java
View File

@ -1,145 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
import org.apache.dolphinscheduler.remote.exceptions.RemoteException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Enumeration;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class IPUtils {
private IPUtils() {
throw new IllegalStateException(IPUtils.class.getName());
}
private static final Logger logger = LoggerFactory.getLogger(IPUtils.class);
private static String localHost = "unknown";
static {
String host = System.getenv("HOSTNAME");
if (isNotEmpty(host)) {
localHost = host;
} else {
try {
String hostName = InetAddress.getLocalHost().getHostName();
if (isNotEmpty(hostName)) {
localHost = hostName;
}
} catch (UnknownHostException e) {
logger.error("get hostName error!", e);
}
}
}
public static String getLocalHost() {
return localHost;
}
public static String getFirstNoLoopbackIP4Address() {
Collection<String> allNoLoopbackIP4Addresses = getNoLoopbackIP4Addresses();
if (allNoLoopbackIP4Addresses.isEmpty()) {
return null;
}
return allNoLoopbackIP4Addresses.iterator().next();
}
public static Collection<String> getNoLoopbackIP4Addresses() {
Collection<String> noLoopbackIP4Addresses = new ArrayList<>();
Collection<InetAddress> allInetAddresses = getAllHostAddress();
for (InetAddress address : allInetAddresses) {
if (!address.isLoopbackAddress() && !address.isSiteLocalAddress()
&& !Inet6Address.class.isInstance(address)) {
noLoopbackIP4Addresses.add(address.getHostAddress());
}
}
if (noLoopbackIP4Addresses.isEmpty()) {
for (InetAddress address : allInetAddresses) {
if (!address.isLoopbackAddress() && !Inet6Address.class.isInstance(address)) {
noLoopbackIP4Addresses.add(address.getHostAddress());
}
}
}
return noLoopbackIP4Addresses;
}
public static Collection<InetAddress> getAllHostAddress() {
try {
Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
Collection<InetAddress> addresses = new ArrayList<>();
while (networkInterfaces.hasMoreElements()) {
NetworkInterface networkInterface = networkInterfaces.nextElement();
Enumeration<InetAddress> inetAddresses = networkInterface.getInetAddresses();
while (inetAddresses.hasMoreElements()) {
InetAddress inetAddress = inetAddresses.nextElement();
addresses.add(inetAddress);
}
}
return addresses;
} catch (SocketException e) {
throw new RemoteException(e.getMessage(), e);
}
}
public static String getIpByHostName(String host) {
InetAddress address = null;
try {
address = InetAddress.getByName(host);
} catch (UnknownHostException e) {
logger.error("get IP error", e);
}
if (address == null) {
return "";
}
return address.getHostAddress();
}
private static boolean isEmpty(final CharSequence cs) {
return cs == null || cs.length() == 0;
}
private static boolean isNotEmpty(final CharSequence cs) {
return !isEmpty(cs);
}
public static boolean isIp(String addr) {
if (addr.length() < 7 || addr.length() > 15 || "".equals(addr)) {
return false;
}
String ipRegex = "([1-9]|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\.(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3}";
Pattern pat = Pattern.compile(ipRegex);
Matcher mat = pat.matcher(addr);
return mat.find();
}
}

4
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.service.log;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.log.GetLogBytesRequestCommand;
@ -31,7 +32,6 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.remote.utils.IPUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -118,7 +118,7 @@ public class LogClientService {
String result = "";
final Host address = new Host(host, port);
try {
if (IPUtils.getLocalHost().equals(host)) {
if (NetUtils.getHost().equals(host)) {
result = LoggerUtils.readWholeFileContent(request.getPath());
} else {
Command command = request.convert2Command();

14
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientServiceTest.java
View File

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.service.log;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand;
@ -26,7 +27,6 @@ import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseComma
import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.remote.utils.IPUtils;
import java.nio.charset.StandardCharsets;
@ -40,7 +40,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@RunWith(PowerMockRunner.class)
@PrepareForTest({LogClientService.class, IPUtils.class, LoggerUtils.class, NettyRemotingClient.class})
@PrepareForTest({LogClientService.class, NetUtils.class, LoggerUtils.class, NettyRemotingClient.class})
public class LogClientServiceTest {
@Test
@ -49,8 +49,8 @@ public class LogClientServiceTest {
int port = 1234;
String path = "/tmp/log";
PowerMockito.mockStatic(IPUtils.class);
PowerMockito.when(IPUtils.getLocalHost()).thenReturn(localMachine);
PowerMockito.mockStatic(NetUtils.class);
PowerMockito.when(NetUtils.getHost()).thenReturn(localMachine);
PowerMockito.mockStatic(LoggerUtils.class);
PowerMockito.when(LoggerUtils.readWholeFileContent(Mockito.anyString())).thenReturn("application_xx_11");
@ -61,12 +61,12 @@ public class LogClientServiceTest {
@Test
public void testViewLogFromRemote() throws Exception {
String localMachine = "LOCAL_MACHINE";
String localMachine = "127.0.0.1";
int port = 1234;
String path = "/tmp/log";
PowerMockito.mockStatic(IPUtils.class);
PowerMockito.when(IPUtils.getLocalHost()).thenReturn(localMachine + "1");
PowerMockito.mockStatic(NetUtils.class);
PowerMockito.when(NetUtils.getHost()).thenReturn(localMachine + "1");
NettyRemotingClient remotingClient = PowerMockito.mock(NettyRemotingClient.class);
PowerMockito.whenNew(NettyRemotingClient.class).withAnyArguments().thenReturn(remotingClient);

1
pom.xml
View File

@ -864,7 +864,6 @@
<include>**/common/utils/DependentUtilsTest.java</include>
<include>**/common/utils/EncryptionUtilsTest.java</include>
<include>**/common/utils/FileUtilsTest.java</include>
<include>**/common/utils/IpUtilsTest.java</include>
<include>**/common/utils/JSONUtilsTest.java</include>
<include>**/common/utils/LoggerUtilsTest.java</include>
<include>**/common/utils/NetUtilsTest.java</include>