jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!575 Asynchronous load and refresh for metadata tree 

Merge pull request !575 from Peikun Chen/feature/metadataTree
This commit is contained in:
i-robot 2021-02-03 16:16:52 +08:00 committed by Gitee
parent 357b804761 7879643015
commit 06d52c34d8
28 changed files with 28897 additions and 125054 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
presto-main/src/main/java/io/prestosql/catalog/DynamicCatalogStore.java
View File

@ -299,6 +299,7 @@ public class DynamicCatalogStore
{
try {
shareCatalogStore.deleteCatalog(catalogName, false);
connectorManager.dropConnection(catalogName);
}
catch (IOException e) {
throw new PrestoException(GENERIC_INTERNAL_ERROR, "Delete catalog failed.", e);

36
presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorConfig.java
View File

@ -36,9 +36,6 @@ public class QueryEditorConfig
private Optional<String> sharedSecret = Optional.empty();
private Duration sessionTimeout = new Duration(1, DAYS);
private Duration executionTimeout = new Duration(15, MINUTES);
private int schemaCacheExpiryMin = 5;
private int previewTableCacheExpiryMin = 20;
private boolean populateSchemaCacheOnStartup = true;
public int getMaxResultCount()
{
@ -146,37 +143,4 @@ public class QueryEditorConfig
{
this.executionTimeout = executionTimeout;
}
@Config("hetu.queryeditor-ui.schema-cache.expiry.min")
public void setSchemaCacheExpiryMin(int schemaCacheExpiryMin)
{
this.schemaCacheExpiryMin = schemaCacheExpiryMin;
}
public int getSchemaCacheExpiryMin()
{
return schemaCacheExpiryMin;
}
@Config("hetu.queryeditor-ui.previewtable-cache.expiry.min")
public void setPreviewTableCacheExpiryMin(int previewTableCacheExpiryMin)
{
this.previewTableCacheExpiryMin = previewTableCacheExpiryMin;
}
public int getPreviewTableCacheExpiryMin()
{
return previewTableCacheExpiryMin;
}
@Config("hetu.queryeditor-ui.schema-cache.prepopulate.enabled")
public void setPopulateSchemaCacheOnStartup(boolean populateSchemaCacheOnStartup)
{
this.populateSchemaCacheOnStartup = populateSchemaCacheOnStartup;
}
public boolean isPopulateSchemaCacheOnStartup()
{
return populateSchemaCacheOnStartup;
}
}

59
presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorUIModule.java
View File

@ -26,15 +26,13 @@ import io.airlift.http.server.HttpServerConfig;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import io.prestosql.client.SocketChannelSocketFactory;
import io.prestosql.connector.DataCenterConnectorManager;
import io.prestosql.metadata.CatalogManager;
import io.prestosql.queryeditorui.execution.ClientSessionFactory;
import io.prestosql.queryeditorui.execution.ExecutionClient;
import io.prestosql.queryeditorui.execution.QueryInfoClient;
import io.prestosql.queryeditorui.execution.QueryRunner.QueryRunnerFactory;
import io.prestosql.queryeditorui.metadata.ColumnCache;
import io.prestosql.queryeditorui.metadata.PreviewTableCache;
import io.prestosql.queryeditorui.metadata.SchemaCache;
import io.prestosql.queryeditorui.metadata.ColumnService;
import io.prestosql.queryeditorui.metadata.PreviewTableService;
import io.prestosql.queryeditorui.metadata.SchemaService;
import io.prestosql.queryeditorui.output.PersistentJobOutputFactory;
import io.prestosql.queryeditorui.output.builders.OutputBuilderFactory;
import io.prestosql.queryeditorui.output.persistors.CSVPersistorFactory;
@ -44,9 +42,9 @@ import io.prestosql.queryeditorui.protocol.ExecutionStatus.ExecutionSuccess;
import io.prestosql.queryeditorui.resources.ConnectorResource;
import io.prestosql.queryeditorui.resources.FilesResource;
import io.prestosql.queryeditorui.resources.LoginResource;
import io.prestosql.queryeditorui.resources.MetadataResource;
import io.prestosql.queryeditorui.resources.QueryResource;
import io.prestosql.queryeditorui.resources.ResultsPreviewResource;
import io.prestosql.queryeditorui.resources.TablesResource;
import io.prestosql.queryeditorui.resources.UIExecuteResource;
import io.prestosql.queryeditorui.resources.UserResource;
import io.prestosql.queryeditorui.security.UiAuthenticator;
@ -69,8 +67,6 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
@ -102,11 +98,14 @@ public class QueryEditorUIModule
jaxrsBinder(binder).bind(FilesResource.class);
jaxrsBinder(binder).bind(QueryResource.class);
jaxrsBinder(binder).bind(ResultsPreviewResource.class);
jaxrsBinder(binder).bind(TablesResource.class);
jaxrsBinder(binder).bind(MetadataResource.class);
jaxrsBinder(binder).bind(ConnectorResource.class);
jaxrsBinder(binder).bind(LoginResource.class);
jaxrsBinder(binder).bind(UserResource.class);
binder.bind(SchemaService.class).in(Scopes.SINGLETON);
binder.bind(ColumnService.class).in(Scopes.SINGLETON);
binder.bind(PreviewTableService.class).in(Scopes.SINGLETON);
binder.bind(ExecutionClient.class).in(Scopes.SINGLETON);
binder.bind(PersistentJobOutputFactory.class).in(Scopes.SINGLETON);
binder.bind(JobHistoryStore.class).to(LocalJobHistoryStore.class).in(Scopes.SINGLETON);
@ -232,48 +231,6 @@ public class QueryEditorUIModule
return new QueryInfoClient(httpClient);
}
@Singleton
@Provides
public SchemaCache provideSchemaCache(QueryRunnerFactory queryRunnerFactory,
CatalogManager catalogManager,
DataCenterConnectorManager dataCenterConnectorManager,
@Named("hetu") ExecutorService executorService,
QueryEditorConfig config)
{
final SchemaCache cache = new SchemaCache(queryRunnerFactory, executorService, config.getSchemaCacheExpiryMin());
if (config.isPopulateSchemaCacheOnStartup()) {
cache.populateCache(catalogManager, dataCenterConnectorManager);
}
return cache;
}
@Singleton
@Provides
public ColumnCache provideColumnCache(QueryRunnerFactory queryRunnerFactory,
@Named("hetu") ExecutorService executorService,
QueryEditorConfig config)
{
return new ColumnCache(queryRunnerFactory, executorService, config.getSchemaCacheExpiryMin());
}
@Singleton
@Provides
public PreviewTableCache providePreviewTableCache(QueryRunnerFactory queryRunnerFactory,
@Named("hetu") ExecutorService executorService,
QueryEditorConfig config)
{
return new PreviewTableCache(queryRunnerFactory,
executorService, config.getPreviewTableCacheExpiryMin());
}
@Singleton
@Named("hetu")
@Provides
public ExecutorService provideCompleterExecutorService()
{
return Executors.newCachedThreadPool(SchemaCache.daemonThreadsNamed("presto-%d"));
}
@Provides
public QueryStore provideQueryStore(QueryEditorConfig queryEditorConfig) throws IOException
{

74
presto-main/src/main/java/io/prestosql/queryeditorui/metadata/ColumnCache.java → presto-main/src/main/java/io/prestosql/queryeditorui/metadata/ColumnService.java
View File

@ -14,72 +14,57 @@
package io.prestosql.queryeditorui.metadata;
import com.google.common.base.Joiner;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import io.airlift.log.Logger;
import io.prestosql.client.Column;
import io.prestosql.client.QueryData;
import io.prestosql.client.StatementClient;
import io.prestosql.queryeditorui.QueryEditorUIModule;
import io.prestosql.queryeditorui.execution.BackgroundCacheLoader;
import io.prestosql.queryeditorui.execution.QueryClient;
import io.prestosql.queryeditorui.execution.QueryRunner;
import io.prestosql.queryeditorui.execution.QueryRunner.QueryRunnerFactory;
import io.prestosql.queryeditorui.security.UiAuthenticator;
import io.prestosql.server.protocol.Query;
import io.prestosql.spi.type.TypeSignature;
import org.joda.time.Duration;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
public class ColumnCache
public class ColumnService
{
private static final Logger log = Logger.get(ColumnCache.class);
private static final Logger log = Logger.get(ColumnService.class);
private static final Joiner FQN_JOINER = Joiner.on('.').skipNulls();
private final LoadingCache<String, List<Column>> tableColumnCache;
private final QueryRunnerFactory queryRunnerFactory;
private final ExecutorService executor;
public ColumnCache(final QueryRunnerFactory queryRunnerFactory,
final ExecutorService executor, int cacheExpiryMin)
@Inject
public ColumnService(final QueryRunnerFactory queryRunnerFactory)
{
this.queryRunnerFactory = requireNonNull(queryRunnerFactory, "queryRunnerFactory session was null!");
this.executor = requireNonNull(executor, "executor was null!");
ListeningExecutorService listeningExecutor = MoreExecutors.listeningDecorator(executor);
BackgroundCacheLoader<String, List<Column>> columnLoader = new BackgroundCacheLoader<String,
List<Column>>(listeningExecutor)
{
@Override
public List<Column> load(String fqTableName)
{
return queryColumns(format("SHOW COLUMNS FROM %s", fqTableName));
}
};
this.tableColumnCache = CacheBuilder.newBuilder()
.expireAfterWrite(cacheExpiryMin, TimeUnit.MINUTES)
.build(columnLoader);
}
private List<Column> queryColumns(String query)
public List<Column> getColumns(String catalogName, String schemaName,
String tableName, HttpServletRequest servletRequest) throws ExecutionException
{
final ImmutableList.Builder<Column> cache = ImmutableList.builder();
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, "lk");
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(60), query);
return queryColumns(FQN_JOINER.join(catalogName, schemaName, tableName), servletRequest);
}
private List<Column> queryColumns(String fqnTableName, HttpServletRequest servletRequest)
{
String user = UiAuthenticator.getUser(servletRequest);
String statement = format("SHOW COLUMNS FROM %s", fqnTableName);
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, user);
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(60), statement);
final ImmutableList.Builder<Column> cache = ImmutableList.builder();
try {
queryClient.executeWith(new Function<StatementClient, Void>() {
@Nullable
@ -105,25 +90,4 @@ public class ColumnCache
return cache.build();
}
public void refreshCache()
{
tableColumnCache.invalidateAll();
}
public void populateCache(final String fqnTableName)
{
requireNonNull(fqnTableName, "fqnTableName is null");
executor.execute(() -> tableColumnCache.refresh(fqnTableName));
}
public List<Column> getColumns(String schemaName, String tableName) throws ExecutionException
{
return tableColumnCache.get(getFqnTableName(schemaName, tableName));
}
public static String getFqnTableName(String schemaName, String tableName)
{
return FQN_JOINER.join(schemaName, tableName);
}
}

131
presto-main/src/main/java/io/prestosql/queryeditorui/metadata/PreviewTableCache.java
View File

@ -1,131 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.metadata;
import com.google.common.base.Joiner;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import io.airlift.log.Logger;
import io.prestosql.client.QueryData;
import io.prestosql.client.StatementClient;
import io.prestosql.queryeditorui.QueryEditorUIModule;
import io.prestosql.queryeditorui.execution.BackgroundCacheLoader;
import io.prestosql.queryeditorui.execution.QueryClient;
import io.prestosql.queryeditorui.execution.QueryRunner;
import io.prestosql.queryeditorui.protocol.Table;
import org.joda.time.Duration;
import javax.annotation.Nullable;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
public class PreviewTableCache
{
private static final Logger log = Logger.get(PreviewTableCache.class);
private static final Joiner FQN_JOINER = Joiner.on('.').skipNulls();
private static final int PREVIEW_LIMIT = 100;
private final LoadingCache<Table, List<List<Object>>> previewTableCache;
private final QueryRunner.QueryRunnerFactory queryRunnerFactory;
public PreviewTableCache(final QueryRunner.QueryRunnerFactory queryRunnerFactory,
final ExecutorService executor, int cacheExpiryMin)
{
this.queryRunnerFactory = requireNonNull(queryRunnerFactory, "queryRunnerFactory session was null!");
ListeningExecutorService listeningExecutor = MoreExecutors.listeningDecorator(executor);
BackgroundCacheLoader<Table, List<List<Object>>> tableLoader =
new BackgroundCacheLoader<Table, List<List<Object>>>(listeningExecutor)
{
@Override
public List<List<Object>> load(Table key)
throws Exception
{
return queryRows(buildQueryWithLimit(key, PREVIEW_LIMIT));
}
};
this.previewTableCache = CacheBuilder.newBuilder()
.expireAfterWrite(cacheExpiryMin, TimeUnit.MINUTES)
.maximumSize(PREVIEW_LIMIT)
.build(tableLoader);
}
private static String buildQueryWithLimit(Table table, int limit)
{
return format("SELECT * FROM %s LIMIT %d",
getFqnTableName(table.getConnectorId(), table.getSchema(), table.getTable()),
limit);
}
private List<List<Object>> queryRows(String query)
{
final ImmutableList.Builder<List<Object>> cache = ImmutableList.builder();
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, "lk");
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(60), query);
try {
queryClient.executeWith(new Function<StatementClient, Void>() {
@Nullable
@Override
public Void apply(StatementClient client)
{
QueryData results = client.currentData();
if (results.getData() != null) {
cache.addAll(results.getData());
}
return null;
}
});
}
catch (QueryClient.QueryTimeOutException e) {
log.error("Caught timeout loading columns", e);
}
return cache.build();
}
public void refreshCache()
{
previewTableCache.invalidateAll();
}
public List<List<Object>> getPreview(final String schema,
final String table)
throws ExecutionException
{
return previewTableCache.get(Table.valueOf(getFqnTableName(schema, table)));
}
public static String getFqnTableName(String connectorId, String schemaName, String tableName)
{
return FQN_JOINER.join(connectorId, schemaName, tableName);
}
public static String getFqnTableName(String schemaName, String tableName)
{
return FQN_JOINER.join(schemaName, tableName);
}
}

86
presto-main/src/main/java/io/prestosql/queryeditorui/metadata/PreviewTableService.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.metadata;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import io.airlift.log.Logger;
import io.prestosql.client.QueryData;
import io.prestosql.client.StatementClient;
import io.prestosql.queryeditorui.QueryEditorUIModule;
import io.prestosql.queryeditorui.execution.QueryClient;
import io.prestosql.queryeditorui.execution.QueryRunner;
import io.prestosql.queryeditorui.security.UiAuthenticator;
import org.joda.time.Duration;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
public class PreviewTableService
{
private static final Logger log = Logger.get(PreviewTableService.class);
private static final Joiner FQN_JOINER = Joiner.on('.').skipNulls();
private static final int PREVIEW_LIMIT = 100;
private final QueryRunner.QueryRunnerFactory queryRunnerFactory;
@Inject
public PreviewTableService(final QueryRunner.QueryRunnerFactory queryRunnerFactory)
{
this.queryRunnerFactory = requireNonNull(queryRunnerFactory, "queryRunnerFactory session was null!");
}
public List<List<Object>> getPreview(String catalogName, String schemaName,
String tableName, HttpServletRequest servletRequest) throws ExecutionException
{
return queryRows(FQN_JOINER.join(catalogName, schemaName, tableName), servletRequest);
}
private List<List<Object>> queryRows(String fqnTableName, HttpServletRequest servletRequest)
{
String user = UiAuthenticator.getUser(servletRequest);
String statement = format("SELECT * FROM %s LIMIT %d", fqnTableName, PREVIEW_LIMIT);
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, user);
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(60), statement);
final ImmutableList.Builder<List<Object>> cache = ImmutableList.builder();
try {
queryClient.executeWith(new Function<StatementClient, Void>() {
@Nullable
@Override
public Void apply(StatementClient client)
{
QueryData results = client.currentData();
if (results.getData() != null) {
cache.addAll(results.getData());
}
return null;
}
});
}
catch (QueryClient.QueryTimeOutException e) {
log.error("Caught timeout loading columns", e);
}
return cache.build();
}
}

268
presto-main/src/main/java/io/prestosql/queryeditorui/metadata/SchemaCache.java
View File

@ -1,268 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.metadata;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.airlift.json.JsonCodec;
import io.airlift.log.Logger;
import io.prestosql.client.JsonResponse;
import io.prestosql.client.QueryData;
import io.prestosql.client.ServerInfo;
import io.prestosql.client.StatementClient;
import io.prestosql.connector.DataCenterConnectorManager;
import io.prestosql.metadata.Catalog;
import io.prestosql.metadata.CatalogManager;
import io.prestosql.queryeditorui.QueryEditorUIModule;
import io.prestosql.queryeditorui.execution.BackgroundCacheLoader;
import io.prestosql.queryeditorui.execution.QueryClient;
import io.prestosql.queryeditorui.execution.QueryRunner;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import org.joda.time.Duration;
import javax.annotation.Nullable;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import static io.airlift.json.JsonCodec.jsonCodec;
import static java.lang.String.format;
import static java.net.HttpURLConnection.HTTP_OK;
import static java.util.Objects.requireNonNull;
public class SchemaCache
implements Closeable
{
private static final Logger log = Logger.get(SchemaCache.class);
private static final Set<String> EXCLUDED_SCHEMAS = Sets.newHashSet("sys");
private final ExecutorService executor;
private final QueryRunner.QueryRunnerFactory queryRunnerFactory;
private final LoadingCache<String, Map<String, List<String>>> schemaTableCache;
private final LoadingCache<String, List<String>> catalogSchemaCache;
public SchemaCache(final QueryRunner.QueryRunnerFactory queryRunnerFactory,
final ExecutorService executor, int cacheExpiryMin)
{
this.queryRunnerFactory = requireNonNull(queryRunnerFactory, "queryRunnerFactory session was null!");
this.executor = requireNonNull(executor, "executor was null!");
ListeningExecutorService listeningExecutor = MoreExecutors.listeningDecorator(executor);
BackgroundCacheLoader<String, Map<String, List<String>>> loader =
new BackgroundCacheLoader<String, Map<String, List<String>>>(listeningExecutor)
{
@Override
public Map<String, List<String>> load(String catalogName)
{
return queryMetadata(format(
"SELECT table_catalog, table_schema, table_name " +
"FROM %s.information_schema.tables " +
"WHERE table_catalog = '%s'",
catalogName, catalogName));
}
};
schemaTableCache = CacheBuilder.newBuilder()
.refreshAfterWrite(cacheExpiryMin, TimeUnit.MINUTES)
.build(loader);
CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder();
cacheBuilder.refreshAfterWrite(cacheExpiryMin, TimeUnit.MINUTES);
catalogSchemaCache = cacheBuilder.build(new CacheLoader<String, List<String>>()
{
@Override
public List<String> load(String key)
throws Exception
{
return querySchemas(format("SELECT schema_name FROM %s.information_schema.schemata", key));
}
});
}
private Map<String, List<String>> queryMetadata(String query)
{
final Map<String, List<String>> cache = new HashMap<>();
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, "lk");
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(60), query);
try {
queryClient.executeWith(new Function<StatementClient, Void>() {
@Nullable
@Override
public Void apply(StatementClient client)
{
QueryData results = client.currentData();
if (results.getData() != null) {
for (List<Object> row : results.getData()) {
String schema = (String) row.get(1);
String table = (String) row.get(2);
if (EXCLUDED_SCHEMAS.contains(schema)) {
continue;
}
List<String> tables = cache.get(schema);
if (tables == null) {
tables = new ArrayList<>();
cache.put(schema, tables);
}
tables.add(table);
}
}
return null;
}
});
}
catch (QueryClient.QueryTimeOutException e) {
log.error("Caught timeout loading columns", e);
}
return ImmutableMap.copyOf(cache);
}
private List<String> querySchemas(String query)
{
final ImmutableList.Builder<String> resultsBuilder = ImmutableList.builder();
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, "lk");
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(120), query);
try {
queryClient.executeWith(new Function<StatementClient, Void>() {
@Nullable
@Override
public Void apply(StatementClient client)
{
QueryData results = client.currentData();
if (results.getData() != null) {
for (List<Object> row : results.getData()) {
String schema = (String) row.get(0);
if (EXCLUDED_SCHEMAS.contains(schema)) {
continue;
}
resultsBuilder.add(schema);
}
}
return null;
}
});
}
catch (QueryClient.QueryTimeOutException e) {
log.error("Caught timeout loading columns", e);
}
return resultsBuilder.build();
}
public void populateCache(CatalogManager catalogManager, DataCenterConnectorManager dataCenterConnectorManager)
{
log.info("Pre-Populating schema cache");
JsonCodec<ServerInfo> serverInfoCodec = jsonCodec(ServerInfo.class);
OkHttpClient httpClient = queryRunnerFactory.getHttpClient();
executor.execute(() -> {
Request request = new Request.Builder()
.url(queryRunnerFactory.getSessionFactory().getServer().toString() + "/v1/info")
.build();
//Wait for startup
while (true) {
try {
JsonResponse<ServerInfo> response = JsonResponse.execute(serverInfoCodec, httpClient, request);
if ((response.getStatusCode() != HTTP_OK) || !response.hasValue() || response.getValue().isStarting()) {
Thread.sleep(1000);
continue;
}
break;
}
catch (InterruptedException e) {
break;
}
}
dataCenterConnectorManager.loadAllDCCatalogs();
List<Catalog> catalogs = catalogManager.getCatalogs();
for (Catalog catalog : catalogs) {
executor.execute(() -> {
try {
getSchemasForCatalog(catalog.getCatalogName());
getSchemaMap(catalog.getCatalogName(), true);
}
catch (ExecutionException e) {
}
});
}
});
}
public void refreshCache(final String catalog)
{
requireNonNull(catalog, "schemaName is null");
executor.execute(() -> {
//Refresh synchronously
schemaTableCache.invalidate(catalog);
schemaTableCache.refresh(catalog);
});
catalogSchemaCache.refresh(catalog);
}
public Set<String> getCatalogs()
{
return schemaTableCache.asMap().keySet();
}
public Map<String, List<String>> getSchemaMap(final String catalog, boolean force)
{
try {
return schemaTableCache.get(catalog);
}
catch (ExecutionException e) {
e.printStackTrace();
return new HashMap<>();
}
}
public List<String> getSchemasForCatalog(String catalog)
throws ExecutionException
{
return catalogSchemaCache.get(catalog);
}
@Override
public void close()
{
executor.shutdownNow();
}
public static ThreadFactory daemonThreadsNamed(String nameFormat)
{
return new ThreadFactoryBuilder().setNameFormat(nameFormat).setDaemon(true).build();
}
}

124
presto-main/src/main/java/io/prestosql/queryeditorui/metadata/SchemaService.java Normal file
View File

@ -0,0 +1,124 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.metadata;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import io.airlift.log.Logger;
import io.prestosql.client.QueryData;
import io.prestosql.client.StatementClient;
import io.prestosql.queryeditorui.QueryEditorUIModule;
import io.prestosql.queryeditorui.execution.QueryClient;
import io.prestosql.queryeditorui.execution.QueryRunner;
import io.prestosql.queryeditorui.protocol.CatalogSchema;
import io.prestosql.queryeditorui.protocol.Table;
import io.prestosql.queryeditorui.security.UiAuthenticator;
import org.joda.time.Duration;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import static java.lang.String.format;
import static java.util.Objects.requireNonNull;
public class SchemaService
{
private static final Logger LOG = Logger.get(SchemaService.class);
private final QueryRunner.QueryRunnerFactory queryRunnerFactory;
@Inject
public SchemaService(final QueryRunner.QueryRunnerFactory queryRunnerFactory)
{
this.queryRunnerFactory = requireNonNull(queryRunnerFactory, "queryRunnerFactory session was null!");
}
public ImmutableList<Table> queryTables(String catalogName, String schemaName, HttpServletRequest servletRequest)
{
String user = UiAuthenticator.getUser(servletRequest);
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, user);
String statement = format("SHOW TABLES FROM %s.%s", catalogName, schemaName);
Set<String> tablesResult = queryStatement(queryRunner, statement);
final ImmutableList.Builder<Table> builder = ImmutableList.builder();
for (String tableName : tablesResult) {
builder.add(new Table(catalogName, schemaName, tableName));
}
return builder.build();
}
public CatalogSchema querySchemas(String catalogName, HttpServletRequest servletRequest)
{
String user = UiAuthenticator.getUser(servletRequest);
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, user);
String statement = format("SHOW SCHEMAS FROM %s", catalogName);
Set<String> schemasResult = queryStatement(queryRunner, statement);
return new CatalogSchema(catalogName, ImmutableList.copyOf(schemasResult));
}
public ImmutableList<CatalogSchema> querySchemas(HttpServletRequest servletRequest)
{
Set<String> catalogs = queryCatalogs(servletRequest);
final ImmutableList.Builder<CatalogSchema> builder = ImmutableList.builder();
for (String catalogName : catalogs) {
builder.add(querySchemas(catalogName, servletRequest));
}
return builder.build();
}
public Set<String> queryCatalogs(HttpServletRequest servletRequest)
{
String user = UiAuthenticator.getUser(servletRequest);
QueryRunner queryRunner = queryRunnerFactory.create(QueryEditorUIModule.UI_QUERY_SOURCE, user);
String statement = "SHOW CATALOGS";
return queryStatement(queryRunner, statement);
}
private Set<String> queryStatement(QueryRunner queryRunner, String statement)
{
QueryClient queryClient = new QueryClient(queryRunner, Duration.standardSeconds(120), statement);
final Set<String> resultSet = new HashSet();
try {
queryClient.executeWith(new Function<StatementClient, Void>() {
@Nullable
@Override
public Void apply(StatementClient client)
{
QueryData results = client.currentData();
if (results.getData() != null) {
for (List<Object> row : results.getData()) {
resultSet.add((String) row.get(0));
}
}
return null;
}
});
}
catch (QueryClient.QueryTimeOutException e) {
LOG.error("Caught timeout loading data", e);
}
return resultSet;
}
}

117
presto-main/src/main/java/io/prestosql/queryeditorui/resources/MetadataResource.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.resources;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import io.prestosql.client.Column;
import io.prestosql.queryeditorui.metadata.ColumnService;
import io.prestosql.queryeditorui.metadata.PreviewTableService;
import io.prestosql.queryeditorui.metadata.SchemaService;
import io.prestosql.queryeditorui.protocol.CatalogSchema;
import io.prestosql.queryeditorui.protocol.Table;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
import java.util.concurrent.ExecutionException;
@Path("/api/metadata")
public class MetadataResource
{
private final SchemaService schemaService;
private final ColumnService columnService;
private final PreviewTableService previewTableService;
@Inject
public MetadataResource(
final SchemaService schemaService,
final ColumnService columnService,
final PreviewTableService previewTableService)
{
this.schemaService = schemaService;
this.columnService = columnService;
this.previewTableService = previewTableService;
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("tables/{catalog}/{schema}")
public Response getTables(
@PathParam("catalog") String catalogName,
@PathParam("schema") String schemaName,
@Context HttpServletRequest servletRequest)
{
ImmutableList<Table> result = schemaService.queryTables(catalogName, schemaName, servletRequest);
return Response.ok(result).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("columns/{catalog}/{schema}/{table}")
public Response getTableColumns(
@PathParam("catalog") String catalogName,
@PathParam("schema") String schemaName,
@PathParam("table") String tableName,
@Context HttpServletRequest servletRequest)
throws ExecutionException
{
List<Column> columnList = columnService.getColumns(catalogName, schemaName, tableName, servletRequest);
return Response.ok(columnList).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("preview/{catalog}/{schema}/{table}")
public Response getTablePreview(
@PathParam("catalog") String catalogName,
@PathParam("schema") String schemaName,
@PathParam("table") String tableName,
@Context HttpServletRequest servletRequest)
throws ExecutionException
{
List<List<Object>> preview = previewTableService.getPreview(catalogName, schemaName, tableName, servletRequest);
return Response.ok(preview).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("schemas")
public Response getSchemas(
@Context HttpServletRequest servletRequest)
throws ExecutionException
{
ImmutableList<CatalogSchema> result = schemaService.querySchemas(servletRequest);
return Response.ok(result).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("schemas/{catalog}")
public Response getSchemasFromCatalog(
@PathParam("catalog") String catalogName,
@Context HttpServletRequest servletRequest)
throws ExecutionException
{
CatalogSchema catalogSchema = schemaService.querySchemas(catalogName, servletRequest);
return Response.ok(catalogSchema).build();
}
}

149
presto-main/src/main/java/io/prestosql/queryeditorui/resources/TablesResource.java
View File

@ -1,149 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.resources;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import io.prestosql.client.Column;
import io.prestosql.connector.DataCenterConnectorManager;
import io.prestosql.metadata.CatalogManager;
import io.prestosql.queryeditorui.metadata.ColumnCache;
import io.prestosql.queryeditorui.metadata.PreviewTableCache;
import io.prestosql.queryeditorui.metadata.SchemaCache;
import io.prestosql.queryeditorui.protocol.CatalogSchema;
import io.prestosql.queryeditorui.protocol.Table;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Path("/api/table")
public class TablesResource
{
private final SchemaCache schemaCache;
private final ColumnCache columnCache;
private final PreviewTableCache previewTableCache;
private final String defaultCatalog;
private final CatalogManager catalogManager;
private final DataCenterConnectorManager dataCenterConnectorManager;
@Inject
public TablesResource(
final SchemaCache schemaCache,
final ColumnCache columnCache,
final PreviewTableCache previewTableCache,
final CatalogManager catalogManager,
final DataCenterConnectorManager dataCenterConnectorManager,
@Named("default-catalog") final String defaultCatalog)
{
this.catalogManager = catalogManager;
this.dataCenterConnectorManager = dataCenterConnectorManager;
this.schemaCache = schemaCache;
this.columnCache = columnCache;
this.previewTableCache = previewTableCache;
this.defaultCatalog = defaultCatalog;
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getTableUpdates(
@QueryParam("catalog") String catalogName)
{
final List<String> catalogsToList = (null != catalogName && !catalogName.isEmpty()) ? ImmutableList.of(catalogName) :
catalogManager.getCatalogs().stream().map(c -> c.getCatalogName()).collect(Collectors.toList());
final ImmutableList.Builder<Table> builder = ImmutableList.builder();
for (String catalog : catalogsToList) {
final Map<String, List<String>> schemaMap = schemaCache.getSchemaMap(catalog, false);
for (Map.Entry<String, List<String>> entry : schemaMap.entrySet()) {
String schema = entry.getKey();
for (String table : entry.getValue()) {
// if (isAuthorizedRead(user, catalog, schema, table)) {
builder.add(new Table(catalog, schema, table));
// }
}
}
}
final List<Table> tables = builder.build();
// final Map<Table, Long> allUsages = usageStore.getUsages(tables);
// final Map<PartitionedTable, DateTime> updateMap = Collections.emptyMap();
return Response.ok(tables).build();
}
// TODO: Make getTableColumns, getTablePartitions and getTablePreview take a 3rd path parameter for catalog
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("{schema}/{tableName}/columns")
public Response getTableColumns(
@PathParam("schema") String schema,
@PathParam("tableName") String tableName)
throws ExecutionException
{
List<Column> columnList = columnCache.getColumns(schema, tableName);
return Response.ok(columnList).build();
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("{schema}/{tableName}/preview")
public Response getTablePreview(
@PathParam("schema") String schema,
@PathParam("tableName") String tableName)
throws ExecutionException
{
// if (isAuthorizedRead(user, defaultCatalog, schema, tableName)) {
final List<List<Object>> preview = previewTableCache.getPreview(schema, tableName);
return Response.ok(preview).build();
// }
// else {
// return Response.status(Response.Status.FORBIDDEN).build();
// }
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("schemas")
public Response getCatalogs(
@QueryParam("force") boolean force)
throws ExecutionException
{
if (force) {
previewTableCache.refreshCache();
columnCache.refreshCache();
}
dataCenterConnectorManager.loadAllDCCatalogs();
List<String> catalogs = catalogManager.getCatalogs().stream().map(c -> c.getCatalogName()).collect(Collectors.toList());
final ImmutableList.Builder<CatalogSchema> builder = ImmutableList.builder();
for (String catalog : catalogs) {
if (force) {
schemaCache.refreshCache(catalog);
}
List<String> schemas = schemaCache.getSchemasForCatalog(catalog);
builder.add(new CatalogSchema(catalog, ImmutableList.copyOf(schemas)));
}
return Response.ok(builder.build()).build();
}
}

4
presto-main/src/main/java/io/prestosql/server/testing/TestingPrestoServer.java
View File

@ -233,10 +233,6 @@ public class TestingPrestoServer
if (coordinator) {
// TODO: enable failure detector
serverProperties.put("failure-detector.enabled", "false");
// Disable UI related background activities to ease the pressure on tests
serverProperties.put("hetu.queryeditor-ui.schema-cache.expiry.min", "60");
serverProperties.put("hetu.queryeditor-ui.schema-cache.prepopulate.enabled", "false");
serverProperties.put("hetu.queryeditor-ui.previewtable-cache.expiry.min", "60");
}
ImmutableList.Builder<Module> modules = ImmutableList.<Module>builder()

8
presto-main/src/main/resources/webapp/assets/stylesheets/hetuui.css
View File

@ -915,7 +915,6 @@ h2 {
padding-left: 10px;*/
}
/* Category opener */
.schema-tree .opener {
@ -936,7 +935,12 @@ h2 {
}
.schema-tree li.category.async>.item>.opener::after {
content: '!';
content: '\f067';
color: gray;
display: block;
transition: all 0.25s;
font-size: 12px;
font-family: FontAwesome;
}
.schema-tree .opener:hover {

16701
presto-main/src/main/resources/webapp/dist/embedded_plan.js vendored
View File

File diff suppressed because one or more lines are too long

6
presto-main/src/main/resources/webapp/dist/headerfooter.js vendored
View File

File diff suppressed because one or more lines are too long

19012
presto-main/src/main/resources/webapp/dist/hetuqueryeditor.js vendored
View File

File diff suppressed because one or more lines are too long

16701
presto-main/src/main/resources/webapp/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

16701
presto-main/src/main/resources/webapp/dist/nodes.js vendored
View File

File diff suppressed because one or more lines are too long

16717
presto-main/src/main/resources/webapp/dist/overview.js vendored
View File

File diff suppressed because one or more lines are too long

16701
presto-main/src/main/resources/webapp/dist/plan.js vendored
View File

File diff suppressed because one or more lines are too long

16701
presto-main/src/main/resources/webapp/dist/query.js vendored
View File

File diff suppressed because one or more lines are too long

16701
presto-main/src/main/resources/webapp/dist/stage.js vendored
View File

File diff suppressed because one or more lines are too long

16699
presto-main/src/main/resources/webapp/dist/worker.js vendored
View File

File diff suppressed because one or more lines are too long

2
presto-main/src/main/resources/webapp/src/package.json
View File

@ -30,7 +30,7 @@
"react-ace": "^9.0.0",
"react-bootstrap": "^1.0.1",
"react-contextmenu": "^2.14.0",
"react-simple-multi-select":"^1.1.0",
"react-simple-multi-select": "^1.1.0",
"react-dom": "^16.8.0",
"react-moment": "^0.9.7",
"react-tabs": "^3.1.1",

183
presto-main/src/main/resources/webapp/src/queryeditor/actions/SchemaActions.js
View File

@ -52,11 +52,9 @@ class SchemaActions {
})
}
fetchSchemas(catalogs, refresh = false) {
return xhr("../api/table/schemas?force=" + refresh).then((data) => {
if (refresh) {
catalogs = [];
}
fetchSchemas(catalogs) {
return xhr("../api/metadata/schemas").then((data) => {
catalogs = [];
for (const [index, entry] of data.entries()) {
let catalog = _.find(catalogs, {name: entry.catalogName});
if (_.isUndefined(catalog)) {
@ -77,38 +75,30 @@ class SchemaActions {
type: dataType.SCHEMA,
catalog: catalog.name,
fqn: "schematree-schema." + catalog.name + "." + schemaName,
children: []
children: () => {
return new Promise((resolve, reject) => {
var schemaTables = xhr("../api/metadata/tables/" + catalog.name + "/" + schemaName).then((data) => {
var tables = [];
for (const [index, entry] of data.entries()) {
var table = {
name: entry.table,
type: dataType.TABLE,
catalog: entry.connectorId,
schema: entry.schema,
fqn: entry.fqn
};
tables.push(table);
}
return tables;
});
// resolves into the children model
resolve(schemaTables);
});
}
};
schemas.push(schema);
}
}
if (!refresh) {
//Remove deleted schemas
//Reverse iterate and mutate array by index.
let existingSchemas = catalog.children;
let schemaIndex = existingSchemas.length - 1;
while (schemaIndex >= 0) {
let currentSchema = existingSchemas[schemaIndex];
let fetchedSchema = entry.schemas.indexOf(currentSchema.name);
if (_.isUndefined(fetchedSchema) || fetchedSchema < 0) {
existingSchemas.splice(schemaIndex, 1);
}
schemaIndex -= 1;
}
}
}
if (!refresh) {
//Remove removed catalogs
//Reverse iterate and mutate array by index.
let index = catalogs.length - 1;
while (index >= 0) {
let currentCatalog = catalogs[index];
let fetchedCatalog = _.find(data, {catalogName: currentCatalog.name});
if (_.isUndefined(fetchedCatalog)) {
catalogs.splice(index, 1);
}
index -= 1;
}
}
return catalogs;
}).then((catalogs) => {
@ -117,65 +107,76 @@ class SchemaActions {
});
}
fetchTables(catalogs) {
return xhr("../api/table").then((data) => {
for (const [index, entry] of data.entries()) {
let catalog = _.find(catalogs, {name: entry.connectorId});
if (_.isUndefined(catalog)) {
catalog = {
name: entry.connectorId,
type: dataType.CATALOG,
fqn: "schematree-catalog." + entry.catalogName,
children: []
};
catalogs.push(catalog);
}
let schemas = catalog.children;
let schema = _.find(schemas, {name: entry.schema});
if (_.isUndefined(schema)) {
schema = {
name: entry.schema,
type: dataType.SCHEMA,
catalog: entry.connectorId,
fqn: "schematree-schema." + catalog.name + "." + schemaName,
children: []
};
schemas.push(schema);
}
let tables = schema.children;
let table = _.find(tables, {name: entry.table});
if (_.isUndefined(table)) {
table = {
name: entry.table,
type: dataType.TABLE,
catalog: entry.connectorId,
schema: entry.schema,
fqn: entry.fqn
};
tables.push(table);
}
}
catalogs.forEach(function (catalog, index, catalogsArray) {
catalog.children.forEach(function (schema, schemaIndex, schemasArray) {
//Remove deleted tables
//Reverse iterate and mutate array by index.
let existingTables = schema.children;
let tableIndex = existingTables.length - 1;
while (tableIndex >= 0) {
let currentTable = existingTables[tableIndex];
let fetchedTable = _.find(data, {connectorId: catalog.name, schema: schema.name, table: currentTable.name});
if (_.isUndefined(fetchedTable)) {
existingTables.splice(tableIndex, 1);
}
tableIndex -= 1;
fetchChildren(catalogs, item) {
if (item.type == dataType.CATALOG) {
return xhr("../api/metadata/schemas/" + item.name).then((data) => {
let catalog = _.find(catalogs, {name: item.name});
if (!_.isUndefined(catalog)) {
let schemas = [];
for (const [index, schemaName] of data.schemas.entries()) {
let schema = {
name: schemaName,
type: dataType.SCHEMA,
catalog: catalog.name,
fqn: "schematree-schema." + catalog.name + "." + schemaName,
children: () => {
return new Promise((resolve, reject) => {
var schemaTables = xhr("../api/metadata/tables/" + catalog.name + "/" + schemaName).then((data) => {
var tables = [];
for (const [index, entry] of data.entries()) {
var table = {
name: entry.table,
type: dataType.TABLE,
catalog: entry.connectorId,
schema: entry.schema,
fqn: entry.fqn
};
tables.push(table);
}
return tables;
});
// resolves into the children model
resolve(schemaTables);
});
}
};
schemas.push(schema);
}
})
})
return catalogs;
}).then((catalogs) => {
this.actions.updateTables(catalogs);
return catalogs;
});
catalog.children = schemas;
}
return catalogs;
}).then((catalogs) => {
this.actions.updateSchemas(catalogs);
return catalogs;
});
}
else if (item.type == dataType.SCHEMA) {
return xhr("../api/metadata/tables/" + item.catalog + "/" + item.name).then((data) => {
let catalog = _.find(catalogs, {name: item.catalog});
if (!_.isUndefined(catalog)) {
let schema = _.find(catalog.children, {name: item.name});
if (!_.isUndefined(schema)) {
var tables = [];
for (const [index, entry] of data.entries()) {
var table = {
name: entry.table,
type: dataType.TABLE,
catalog: entry.connectorId,
schema: entry.schema,
fqn: entry.fqn
};
tables.push(table);
}
schema.children = tables;
}
}
catalog.children = [...catalog.children];
return catalogs;
}).then((catalogs) => {
this.actions.updateTables(catalogs);
return catalogs;
});
}
}
}

2
presto-main/src/main/resources/webapp/src/queryeditor/components/RunsTable.jsx
View File

@ -298,7 +298,7 @@ let CellRenderers = {
let run = rowData._run;
let link = (run.infoUri != null && run.infoUri != "") ? run.infoUri : "#";
if (run.state === RunStateConstants.FAILED) {
return (<span><span className="label label-danger" style={{minWidth:"61px"}}><a href={link} target="_blank">FAILED</a></span><a href="javascript:void(0)" title="Queries may fail or be cancelled if they exceed configured resources limits.This includes limits on memory consumption and result file size"><i className="fa fa-info" style={{top:'3px',color:'#d5534f',marginLeft:'5px'}}></i></a></span>);
return (<span><span className="label label-danger" style={{minWidth:"61px"}}><a href={link} target="_blank">FAILED</a></span><a href="#" title="Queries may fail or be cancelled if they exceed configured resources limits.This includes limits on memory consumption and result file size"><i className="fa fa-info" style={{top:'3px',color:'#d5534f',marginLeft:'5px'}}></i></a></span>);
} else if (run.state === RunStateConstants.FINISHED) {
return (<span className="label label-success" style={{minWidth:"61px"}}><a href={link} target="_blank">{run.state}</a></span>);
} else if (run.state === RunStateConstants.QUEUED) {

57
presto-main/src/main/resources/webapp/src/queryeditor/components/SchemaTree.jsx
View File

@ -45,7 +45,7 @@ function getIcon(type) {
}
function renderItem(tree, item) {
let style = (item.children == undefined || item.children.length == 0) ? {marginLeft: "14.5px"} : {};
let style = (item.children == undefined || item.children instanceof Array && item.children.length == 0) ? {marginLeft: "14.5px"} : {};
let tableStyle = {};
Object.assign(tableStyle, style, {cursor: "pointer"})
let favorite = tree.isFavorite(item);
@ -118,6 +118,11 @@ function renderItem(tree, item) {
<i className="icon fa fa-star valign-middle contextmenu-icons favorite"/><span>Add to Favorites</span>
</MenuItem>
}
<MenuItem data={{item: item, tree: tree}} onClick={(e, data) => {
tree.refreshItem(item);
}}>
<i className="icon fa fa-refresh valign-middle"/><span>Refresh</span>
</MenuItem>
{item.type == dataType.CATALOG ?
<MenuItem data={{item: item, tree: tree}} onClick={(e, data) => {
tree.deleteCatalog(item);
@ -190,36 +195,32 @@ class SchemaTree extends React.Component {
tables: []
}
this.updateTree = this.updateTree.bind(this);
this.refresh = this.refresh.bind(this);
this.selectTable = this.selectTable.bind(this);
this.unselectTable = this.unselectTable.bind(this);
this.addToFavorites = this.addToFavorites.bind(this);
this.removeFromFavorites = this.removeFromFavorites.bind(this);
this.isFavorite = this.isFavorite.bind(this);
this.reloadItem = this.reloadItem.bind(this);
this.refreshItem = this.refreshItem.bind(this);
this.deleteCatalog = this.deleteCatalog.bind(this);
}
updateTree(refresh = false) {
clearTimeout(this.timer);
SchemaActions.fetchSchemas(this.state.model, refresh).then((catalogs) => {
return SchemaActions.fetchTables(catalogs);
}).then((catalogs) => {
let state = this.state;
if (refresh) {
state.model = [];
this.setState(state);
state = this.state;
}
state.model = catalogs;
this.setState(state);
}).then(() => {
this.timer = setTimeout(this.updateTree, 30000)
updateTree() {
SchemaActions.fetchSchemas(this.state.model).then((catalogs) => {
this.state.model = [];
this.setState(this.state);
this.state.model = catalogs;
this.setState(this.state);
});
}
refresh() {
this.updateTree(true)
refreshItem(item) {
SchemaActions.fetchChildren(this.state.model, item).then((catalogs) => {
setTimeout( () => {
this.state.model = [...catalogs];
this.setState(this.state);
}, 100);
});
}
componentDidMount() {
@ -227,7 +228,6 @@ class SchemaTree extends React.Component {
}
componentWillUnmount() {
clearTimeout(this.timer)
}
getInitialModel() {
@ -265,9 +265,8 @@ class SchemaTree extends React.Component {
} else {
alert("Error while delete catalog:" + res.message.split('\n', 1)[0]);
}
} else {
this.refresh();
this.updateTree();
}
})
}
@ -280,7 +279,7 @@ class SchemaTree extends React.Component {
let favoriteCatalog = _.find(this.favourites.catalogs, catalog);
if (_.isUndefined(favoriteCatalog)) {
this.favourites.catalogs.push(catalog);
this.refreshItem(item);
this.reloadItem(item);
}
return;
}
@ -289,7 +288,7 @@ class SchemaTree extends React.Component {
let favoriteSchema = _.find(this.favourites.schemas, schema);
if (_.isUndefined(favoriteSchema)) {
this.favourites.schemas.push(schema);
this.refreshItem(item);
this.reloadItem(item);
}
return;
}
@ -297,7 +296,7 @@ class SchemaTree extends React.Component {
let favoriteTable = _.find(this.favourites.tables, table);
if (_.isUndefined(favoriteTable)) {
this.favourites.tables.push(table);
this.refreshItem(item);
this.reloadItem(item);
}
}
@ -324,7 +323,7 @@ class SchemaTree extends React.Component {
this.favourites.tables.splice(index, 1);
}
}
this.refreshItem(item);
this.reloadItem(item);
}
/**
@ -365,7 +364,7 @@ class SchemaTree extends React.Component {
return {found: false, self: false};
}
refreshItem(item) {
reloadItem(item) {
let model = this.state.model;
this.state.model = [];
this.setState(this.state);
@ -378,10 +377,10 @@ class SchemaTree extends React.Component {
renderButtons() {
return (
<div className={"flex flex-row"} style={{justifyContent: 'space-between'}}>
<AddCatalogContainer refreshCallback={this.refresh}/>
<AddCatalogContainer refreshCallback={this.updateTree}/>
<button className={"btn btn-default"}
style={{margin: "10px"}}
onClick={this.refresh}>
onClick={this.updateTree}>
<i className="fa fa-refresh" style={{top:'3px',color:'#39b0d2',marginRight:'0'}}></i>
</button>
</div>

2
presto-main/src/main/resources/webapp/src/queryeditor/stores/TableStore.js
View File

@ -120,7 +120,7 @@ class TableStore {
// Enrich the table with some extra data (active status and url)
table = _.extend(table, {
active: true,
url: `../api/table/${FQN.schema(table.name)}/${FQN.table(table.name)}`,
url: `${FQN.catalog(table.name)}/${FQN.simpleSchema(table.name)}/${FQN.table(table.name)}`,
partitions: [],
});

8
presto-main/src/main/resources/webapp/src/queryeditor/utils/TableApiUtils.js
View File

@ -13,10 +13,10 @@
*/
import xhr from './xhr';
const fetchColumData = (table) => xhr(`${table.url}/columns`);
const fetchColumData = (table) => xhr(`/api/metadata/columns/${table.url}`);
const fetchPreviewData = (table, partition = {}) => {
let url = `${table.url}/preview`;
let url = `/api/metadata/preview/${table.url}`;
if (partition.name && partition.value) {
url += '?' +
`partitionName=${partition.name}&` +
@ -26,7 +26,7 @@ const fetchPreviewData = (table, partition = {}) => {
return xhr(url);
};
const fetchPartitionData = (table) => xhr(`${table.url}/partitions`);
const fetchPartitionData = (table) => xhr(`/api/metadata/partitions/${table.url}`);
export default {
fetchTableData(table) {
@ -45,6 +45,6 @@ export default {
},
fetchTables() {
return xhr('../api/table?query=a');
return xhr('../api/metadata/tables');
}
};