Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](audit_loader) fix that old external audit loader plugin not work because of incompatibility with new audit plugin (#40565) #41400

Merged
merged 1 commit into from
Sep 28, 2024
Merged

[fix](audit_loader) fix that old external audit loader plugin not work because of incompatibility with new audit plugin (#40565) #41400

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.analysis.TypeDef;
import org.apache.doris.common.UserException;
import org.apache.doris.plugin.audit.AuditLoaderPlugin;
import org.apache.doris.plugin.audit.AuditLoader;
import org.apache.doris.statistics.StatisticConstants;

import com.google.common.collect.Lists;
Expand Down Expand Up @@ -99,7 +99,7 @@ public static List<ColumnDef> getCopiedSchema(String tblName) throws UserExcepti
schema = COL_STATS_SCHEMA;
} else if (tblName.equals(StatisticConstants.HISTOGRAM_TBL_NAME)) {
schema = HISTO_STATS_SCHEMA;
} else if (tblName.equals(AuditLoaderPlugin.AUDIT_LOG_TABLE)) {
} else if (tblName.equals(AuditLoader.AUDIT_LOG_TABLE)) {
schema = AUDIT_SCHEMA;
} else {
throw new UserException("Unknown internal table name: " + tblName);
Expand Down
10 changes: 5 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.ha.FrontendNodeType;
import org.apache.doris.plugin.audit.AuditLoaderPlugin;
import org.apache.doris.plugin.audit.AuditLoader;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.statistics.util.StatisticsUtil;

Expand Down Expand Up @@ -92,7 +92,7 @@ public void run() {
Database database = op.get();
modifyTblReplicaCount(database, StatisticConstants.STATISTIC_TBL_NAME);
modifyTblReplicaCount(database, StatisticConstants.HISTOGRAM_TBL_NAME);
modifyTblReplicaCount(database, AuditLoaderPlugin.AUDIT_LOG_TABLE);
modifyTblReplicaCount(database, AuditLoader.AUDIT_LOG_TABLE);
}

@VisibleForTesting
Expand Down Expand Up @@ -223,7 +223,7 @@ private static CreateTableStmt buildHistogramTblStmt() throws UserException {

private static CreateTableStmt buildAuditTblStmt() throws UserException {
TableName tableName = new TableName("",
FeConstants.INTERNAL_DB_NAME, AuditLoaderPlugin.AUDIT_LOG_TABLE);
FeConstants.INTERNAL_DB_NAME, AuditLoader.AUDIT_LOG_TABLE);

String engineName = "olap";
ArrayList<String> dupKeys = Lists.newArrayList("query_id", "time", "client_ip");
Expand All @@ -246,7 +246,7 @@ private static CreateTableStmt buildAuditTblStmt() throws UserException {
}
};
CreateTableStmt createTableStmt = new CreateTableStmt(true, false,
tableName, InternalSchema.getCopiedSchema(AuditLoaderPlugin.AUDIT_LOG_TABLE),
tableName, InternalSchema.getCopiedSchema(AuditLoader.AUDIT_LOG_TABLE),
engineName, keysDesc, partitionDesc, distributionDesc,
properties, null, "Doris internal audit table, DO NOT MODIFY IT", null);
StatisticsUtil.analyze(createTableStmt);
Expand Down Expand Up @@ -288,7 +288,7 @@ private boolean created() {
}

// 3. check audit table
optionalStatsTbl = db.getTable(AuditLoaderPlugin.AUDIT_LOG_TABLE);
optionalStatsTbl = db.getTable(AuditLoader.AUDIT_LOG_TABLE);
if (!optionalStatsTbl.isPresent()) {
return false;
}
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadRecordMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.plugin.audit.AuditEvent;
import org.apache.doris.plugin.audit.AuditEvent.EventType;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditEvent.EventType;
import org.apache.doris.plugin.audit.StreamLoadAuditEvent;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.Backend;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import org.apache.doris.load.BrokerFileGroupAggInfo;
import org.apache.doris.load.EtlJobType;
import org.apache.doris.load.FailMsg;
import org.apache.doris.plugin.audit.AuditEvent;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.audit.LoadAuditEvent;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
Expand Down
2 changes: 1 addition & 1 deletion ...apache/doris/plugin/audit/AuditEvent.java → ...a/org/apache/doris/plugin/AuditEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.plugin.audit;
package org.apache.doris.plugin;


import java.lang.annotation.Retention;
Expand Down
2 changes: 0 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/plugin/AuditPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

package org.apache.doris.plugin;

import org.apache.doris.plugin.audit.AuditEvent;

/**
* Audit plugin interface describe.
*/
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/plugin/PluginMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.plugin.PluginInfo.PluginType;
import org.apache.doris.plugin.PluginLoader.PluginStatus;
import org.apache.doris.plugin.audit.AuditLoaderPlugin;
import org.apache.doris.plugin.audit.AuditLoader;
import org.apache.doris.plugin.audit.AuditLogBuilder;
import org.apache.doris.plugin.dialect.HttpDialectConverterPlugin;

Expand Down Expand Up @@ -113,7 +113,7 @@ private void initBuiltinPlugins() {
}

// AuditLoader: log audit log to internal table
AuditLoaderPlugin auditLoaderPlugin = new AuditLoaderPlugin();
AuditLoader auditLoaderPlugin = new AuditLoader();
if (!registerBuiltinPlugin(auditLoaderPlugin.getPluginInfo(), auditLoaderPlugin)) {
LOG.warn("failed to register audit log builder");
}
Expand Down
9 changes: 5 additions & 4 deletions ...doris/plugin/audit/AuditLoaderPlugin.java → ...pache/doris/plugin/audit/AuditLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.common.util.DigitalVersion;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditPlugin;
import org.apache.doris.plugin.Plugin;
import org.apache.doris.plugin.PluginContext;
Expand Down Expand Up @@ -47,8 +48,8 @@
/*
* This plugin will load audit log to specified doris table at specified interval
*/
public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
private static final Logger LOG = LogManager.getLogger(AuditLoaderPlugin.class);
public class AuditLoader extends Plugin implements AuditPlugin {
private static final Logger LOG = LogManager.getLogger(AuditLoader.class);

public static final String AUDIT_LOG_TABLE = "audit_log";

Expand All @@ -70,10 +71,10 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {

private final PluginInfo pluginInfo;

public AuditLoaderPlugin() {
public AuditLoader() {
pluginInfo = new PluginInfo(PluginMgr.BUILTIN_PLUGIN_PREFIX + "AuditLoader", PluginType.AUDIT,
"builtin audit loader, to load audit log to internal table", DigitalVersion.fromString("2.1.0"),
DigitalVersion.fromString("1.8.31"), AuditLoaderPlugin.class.getName(), null, null);
DigitalVersion.fromString("1.8.31"), AuditLoader.class.getName(), null, null);
}

public PluginInfo getPluginInfo() {
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLogBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,14 @@
import org.apache.doris.common.AuditLog;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.DigitalVersion;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditEvent.AuditField;
import org.apache.doris.plugin.AuditEvent.EventType;
import org.apache.doris.plugin.AuditPlugin;
import org.apache.doris.plugin.Plugin;
import org.apache.doris.plugin.PluginInfo;
import org.apache.doris.plugin.PluginInfo.PluginType;
import org.apache.doris.plugin.PluginMgr;
import org.apache.doris.plugin.audit.AuditEvent.AuditField;
import org.apache.doris.plugin.audit.AuditEvent.EventType;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public class AuditStreamLoader {
public AuditStreamLoader() {
this.hostPort = "127.0.0.1:" + Config.http_port;
this.db = FeConstants.INTERNAL_DB_NAME;
this.auditLogTbl = AuditLoaderPlugin.AUDIT_LOG_TABLE;
this.auditLogTbl = AuditLoader.AUDIT_LOG_TABLE;
this.auditLogLoadUrlStr = String.format(loadUrlPattern, hostPort, db, auditLogTbl);
// currently, FE identity is FE's IP, so we replace the "." in IP to make it suitable for label
this.feIdentity = hostPort.replaceAll("\\.", "_").replaceAll(":", "_");
Expand Down
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/plugin/audit/LoadAuditEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.doris.plugin.audit;

import org.apache.doris.plugin.AuditEvent;

public class LoadAuditEvent extends AuditEvent {

@AuditField(value = "JobId")
Expand Down
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/plugin/audit/StreamLoadAuditEvent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.doris.plugin.audit;

import org.apache.doris.plugin.AuditEvent;

public class StreamLoadAuditEvent extends AuditEvent {

@AuditField(value = "Label")
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@

package org.apache.doris.qe;

import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditPlugin;
import org.apache.doris.plugin.Plugin;
import org.apache.doris.plugin.PluginInfo.PluginType;
import org.apache.doris.plugin.PluginMgr;
import org.apache.doris.plugin.audit.AuditEvent;

import com.google.common.collect.Queues;
import org.apache.logging.log4j.LogManager;
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.plugin.audit.AuditEvent.AuditEventBuilder;
import org.apache.doris.plugin.audit.AuditEvent.EventType;
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
import org.apache.doris.plugin.AuditEvent.EventType;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.service.FrontendOptions;

Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.plsql.Exec;
import org.apache.doris.plsql.executor.PlSqlOperation;
import org.apache.doris.plugin.audit.AuditEvent.AuditEventBuilder;
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
import org.apache.doris.proto.Types;
import org.apache.doris.resource.Tag;
import org.apache.doris.service.arrowflight.results.FlightSqlChannel;
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.plugin.audit.AuditEvent;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.thrift.TQueryStatistics;
import org.apache.doris.thrift.TReportWorkloadRuntimeStatusParams;

Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.plugin.audit.AuditLoaderPlugin;
import org.apache.doris.plugin.audit.AuditLoader;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.utframe.TestWithFeService;

Expand Down Expand Up @@ -54,11 +54,11 @@ public void testModifyTblReplicaCount() throws AnalysisException {
.getInternalCatalog().getDbNullable(FeConstants.INTERNAL_DB_NAME);
InternalSchemaInitializer.modifyTblReplicaCount(db, StatisticConstants.STATISTIC_TBL_NAME);
InternalSchemaInitializer.modifyTblReplicaCount(db, StatisticConstants.HISTOGRAM_TBL_NAME);
InternalSchemaInitializer.modifyTblReplicaCount(db, AuditLoaderPlugin.AUDIT_LOG_TABLE);
InternalSchemaInitializer.modifyTblReplicaCount(db, AuditLoader.AUDIT_LOG_TABLE);

checkReplicationNum(db, StatisticConstants.STATISTIC_TBL_NAME);
checkReplicationNum(db, StatisticConstants.HISTOGRAM_TBL_NAME);
checkReplicationNum(db, AuditLoaderPlugin.AUDIT_LOG_TABLE);
checkReplicationNum(db, AuditLoader.AUDIT_LOG_TABLE);
}

private void checkReplicationNum(Database db, String tblName) throws AnalysisException {
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@

import org.apache.doris.catalog.Env;
import org.apache.doris.common.util.DigitalVersion;
import org.apache.doris.plugin.AuditEvent;
import org.apache.doris.plugin.AuditEvent.EventType;
import org.apache.doris.plugin.PluginInfo;
import org.apache.doris.plugin.audit.AuditEvent;
import org.apache.doris.plugin.audit.AuditEvent.EventType;
import org.apache.doris.plugin.audit.AuditLogBuilder;
import org.apache.doris.utframe.UtFrameUtils;

Expand Down
Loading