Skip to content

Commit

Permalink
Add session property for skip view materialization BigQuery
Browse files Browse the repository at this point in the history
  • Loading branch information
ebyhr committed May 16, 2022
1 parent 8083165 commit 52f966b
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,22 @@
package io.trino.plugin.bigquery;

import io.airlift.log.Logger;
import io.trino.plugin.base.session.SessionPropertiesProvider;
import io.trino.spi.connector.Connector;
import io.trino.spi.connector.ConnectorMetadata;
import io.trino.spi.connector.ConnectorPageSourceProvider;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.connector.ConnectorSplitManager;
import io.trino.spi.connector.ConnectorTransactionHandle;
import io.trino.spi.session.PropertyMetadata;
import io.trino.spi.transaction.IsolationLevel;

import javax.inject.Inject;

import java.util.List;
import java.util.Set;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED;
import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports;
import static java.util.Objects.requireNonNull;
Expand All @@ -36,16 +42,21 @@ public class BigQueryConnector
private final BigQueryMetadata metadata;
private final BigQuerySplitManager splitManager;
private final BigQueryPageSourceProvider pageSourceProvider;
private final List<PropertyMetadata<?>> sessionProperties;

@Inject
public BigQueryConnector(
BigQueryMetadata metadata,
BigQuerySplitManager splitManager,
BigQueryPageSourceProvider pageSourceProvider)
BigQueryPageSourceProvider pageSourceProvider,
Set<SessionPropertiesProvider> sessionPropertiesProviders)
{
this.metadata = requireNonNull(metadata, "metadata is null");
this.splitManager = requireNonNull(splitManager, "splitManager is null");
this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null");
this.sessionProperties = requireNonNull(sessionPropertiesProviders, "sessionPropertiesProviders is null").stream()
.flatMap(sessionPropertiesProvider -> sessionPropertiesProvider.getSessionProperties().stream())
.collect(toImmutableList());
}

@Override
Expand Down Expand Up @@ -73,4 +84,10 @@ public ConnectorPageSourceProvider getPageSourceProvider()
{
return pageSourceProvider;
}

@Override
public List<PropertyMetadata<?>> getSessionProperties()
{
return sessionProperties;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import io.airlift.configuration.AbstractConfigurationAwareModule;
import io.trino.plugin.base.session.SessionPropertiesProvider;
import io.trino.spi.NodeManager;

import static com.google.inject.multibindings.Multibinder.newSetBinder;
import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
import static io.airlift.configuration.ConfigBinder.configBinder;

Expand Down Expand Up @@ -52,6 +54,7 @@ protected void setup(Binder binder)
binder.bind(BigQueryPageSourceProvider.class).in(Scopes.SINGLETON);
binder.bind(ViewMaterializationCache.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(BigQueryConfig.class);
newSetBinder(binder, SessionPropertiesProvider.class).addBinding().to(BigQuerySessionProperties.class).in(Scopes.SINGLETON);
}

@Provides
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.bigquery;

import com.google.common.collect.ImmutableList;
import io.trino.plugin.base.session.SessionPropertiesProvider;
import io.trino.spi.connector.ConnectorSession;
import io.trino.spi.session.PropertyMetadata;

import javax.inject.Inject;

import java.util.List;

import static io.trino.spi.session.PropertyMetadata.booleanProperty;

public final class BigQuerySessionProperties
implements SessionPropertiesProvider
{
public static final String SKIP_VIEW_MATERIALIZATION = "skip_view_materialization";

private final List<PropertyMetadata<?>> sessionProperties;

@Inject
public BigQuerySessionProperties(BigQueryConfig config)
{
sessionProperties = ImmutableList.<PropertyMetadata<?>>builder()
.add(booleanProperty(
SKIP_VIEW_MATERIALIZATION,
"Skip materializing views",
config.isSkipViewMaterialization(),
false))
.build();
}

@Override
public List<PropertyMetadata<?>> getSessionProperties()
{
return sessionProperties;
}

public static boolean isSkipViewMaterialization(ConnectorSession session)
{
return session.getProperty(SKIP_VIEW_MATERIALIZATION, Boolean.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import static com.google.cloud.bigquery.TableDefinition.Type.VIEW;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_FAILED_TO_EXECUTE_QUERY;
import static io.trino.plugin.bigquery.BigQuerySessionProperties.isSkipViewMaterialization;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
Expand All @@ -61,7 +62,6 @@ public class BigQuerySplitManager
private final Optional<Integer> parallelism;
private final boolean viewEnabled;
private final Duration viewExpiration;
private final boolean skipViewMaterialization;
private final NodeManager nodeManager;

@Inject
Expand All @@ -78,7 +78,6 @@ public BigQuerySplitManager(
this.parallelism = config.getParallelism();
this.viewEnabled = config.isViewsEnabled();
this.viewExpiration = config.getViewExpireDuration();
this.skipViewMaterialization = config.isSkipViewMaterialization();
this.nodeManager = requireNonNull(nodeManager, "nodeManager cannot be null");
}

Expand Down Expand Up @@ -120,7 +119,7 @@ private List<BigQuerySplit> readFromBigQuery(ConnectorSession session, TableDefi
// Storage API doesn't support reading materialized views
return ImmutableList.of(BigQuerySplit.forViewStream(columns, filter));
}
if (skipViewMaterialization && type == VIEW) {
if (isSkipViewMaterialization(session) && type == VIEW) {
return ImmutableList.of(BigQuerySplit.forViewStream(columns, filter));
}
ReadSession readSession = new ReadSessionCreator(bigQueryClientFactory, bigQueryReadClientFactory, viewEnabled, viewExpiration)
Expand Down

0 comments on commit 52f966b

Please sign in to comment.