Skip to content

Commit

Permalink
Support scan table with mob cf
Browse files Browse the repository at this point in the history
  • Loading branch information
meiyi committed May 15, 2019
1 parent 247c79d commit 19f9ce9
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 39 deletions.
12 changes: 4 additions & 8 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HDFSAclController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
Expand Down Expand Up @@ -113,14 +112,11 @@ public Optional<MasterObserver> getMasterObserver() {
public void preMasterInitialization(final ObserverContext<MasterCoprocessorEnvironment> c)
throws IOException {
if (isHdfsAclEnabled(c.getEnvironment().getConfiguration())) {
MasterServices masterServices = null;
MasterCoprocessorEnvironment mEnv = c.getEnvironment();
if (mEnv instanceof HasMasterServices) {
masterServices = ((HasMasterServices) mEnv).getMasterServices();
}
if (masterServices == null) {
throw new RuntimeException("master services can not be null");
if (!(mEnv instanceof HasMasterServices)) {
throw new IOException("Does not implement HMasterServices");
}
MasterServices masterServices = ((HasMasterServices) mEnv).getMasterServices();
hdfsAclHelper =
new HDFSAclHelper(masterServices.getConfiguration(), masterServices.getConnection());
pathHelper = hdfsAclHelper.getPathHelper();
Expand Down Expand Up @@ -162,7 +158,7 @@ public void postCompletedCreateTableAction(ObserverContext<MasterCoprocessorEnvi
if (hdfsAclHelper != null && !desc.getTableName().isSystemTable()) {
TableName tableName = desc.getTableName();
Path[] paths = new Path[] { pathHelper.getTmpTableDir(tableName),
pathHelper.getTableDir(tableName), pathHelper.getArchiveTableDir(tableName) };
pathHelper.getDataTableDir(tableName), pathHelper.getArchiveTableDir(tableName) };
for (Path path : paths) {
if (!fs.exists(path)) {
fs.mkdirs(path);
Expand Down
62 changes: 31 additions & 31 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/HDFSAclHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
Expand All @@ -51,7 +50,7 @@
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
Expand Down Expand Up @@ -116,8 +115,9 @@ public void setCommonDirPermission() throws IOException {
// The {root/.tmp} and {root/.tmp/data} directories are created to make global user HDFS
// acls can be inherited.
Path[] paths = new Path[] { pathHelper.getRootDir(), pathHelper.getDataDir(),
pathHelper.getTmpDir(), pathHelper.getTmpDataDir(), pathHelper.getArchiveDir(),
pathHelper.getArchiveDataDir(), pathHelper.getSnapshotRootDir() };
pathHelper.getMobDir(), pathHelper.getMobDataDir(), pathHelper.getTmpDir(),
pathHelper.getTmpDataDir(), pathHelper.getArchiveDir(), pathHelper.getArchiveDataDir(),
pathHelper.getSnapshotRootDir() };
for (Path path : paths) {
if (!fs.exists(path)) {
fs.mkdirs(path);
Expand All @@ -133,27 +133,6 @@ public void setCommonDirPermission() throws IOException {
}
}

public boolean grant(byte[] entry, Set<String> users) {
try {
Pair<List<HDFSAclOperation>, List<HDFSAclOperation>> operations;
if (PermissionStorage.isNamespaceEntry(entry)) {
String namespace = Bytes.toString(PermissionStorage.fromNamespaceEntry(entry));
operations = getNamespaceHdfsAclOperations(users, HDFSAclOperation.OperationType.MODIFY,
namespace, new HashSet<>(0));
} else if (PermissionStorage.isGlobalEntry(entry)) {
operations = getGlobalHdfsAclOperations(users, HDFSAclOperation.OperationType.MODIFY,
new HashSet<>(0), new HashSet<>());
} else {
operations = getTableHdfsAclOperations(users, HDFSAclOperation.OperationType.MODIFY,
TableName.valueOf(entry));
}
setHDFSAcl(operations);
return true;
} catch (Exception e) {
return false;
}
}

/**
* Set acl when grant user permission
* @param userPerm the user and permission
Expand Down Expand Up @@ -288,7 +267,7 @@ public void addTableAcl(TableName tableName, String user) {
// set access and default HDFS acl for table dir
fs.modifyAclEntries(pathHelper.getTmpTableDir(tableName), aclEntries);
fs.modifyAclEntries(pathHelper.getArchiveTableDir(tableName), aclEntries);
Path tableDir = pathHelper.getTableDir(tableName);
Path tableDir = pathHelper.getDataTableDir(tableName);
HDFSAclOperation operation = new HDFSAclOperation(fs, tableDir, Sets.newHashSet(user),
HDFSAclOperation.OperationType.MODIFY, READ_EXECUTE, true, new ArrayList<>(0));
setSingleHDFSAcl(operation).get();
Expand Down Expand Up @@ -428,7 +407,7 @@ private Pair<List<HDFSAclOperation>, List<HDFSAclOperation>> getTableHdfsAclOper
*/
private List<Path> getDefaultGlobalPath() {
return Lists.newArrayList(pathHelper.getTmpDataDir(), pathHelper.getDataDir(),
pathHelper.getArchiveDataDir());
pathHelper.getMobDataDir(), pathHelper.getArchiveDataDir());
}

/**
Expand All @@ -440,8 +419,9 @@ private List<Path> getDefaultGlobalPath() {
*/
private List<Path> getDefaultNamespacePath(String namespace, boolean includeNamespaceSnapshot)
throws IOException {
List<Path> paths = Lists.newArrayList(pathHelper.getTmpNsDir(namespace),
pathHelper.getDataNsDir(namespace), pathHelper.getArchiveNsDir(namespace));
List<Path> paths =
Lists.newArrayList(pathHelper.getTmpNsDir(namespace), pathHelper.getDataNsDir(namespace),
pathHelper.getMobDataNsDir(namespace), pathHelper.getArchiveNsDir(namespace));
if (includeNamespaceSnapshot) {
paths.addAll(getNamespaceSnapshotPath(namespace));
}
Expand All @@ -463,7 +443,8 @@ private List<Path> getNamespaceSnapshotPath(String namespace) throws IOException
private List<Path> getDefaultTablePath(TableName tableName, boolean includeTableSnapshotPath)
throws IOException {
List<Path> paths = Lists.newArrayList(pathHelper.getTmpTableDir(tableName),
pathHelper.getTableDir(tableName), pathHelper.getArchiveTableDir(tableName));
pathHelper.getDataTableDir(tableName), pathHelper.getMobTableDir(tableName),
pathHelper.getArchiveTableDir(tableName));
if (includeTableSnapshotPath) {
paths.addAll(getTableSnapshotPath(tableName));
}
Expand Down Expand Up @@ -660,6 +641,7 @@ protected static final class PathHelper {
Path rootDir;
Path tmpDataDir;
Path dataDir;
Path mobDataDir;
Path archiveDataDir;
Path snapshotDir;

Expand All @@ -669,6 +651,7 @@ protected static final class PathHelper {
tmpDataDir = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY),
HConstants.BASE_NAMESPACE_DIR);
dataDir = new Path(rootDir, HConstants.BASE_NAMESPACE_DIR);
mobDataDir = new Path(MobUtils.getMobHome(rootDir), HConstants.BASE_NAMESPACE_DIR);
archiveDataDir = new Path(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY),
HConstants.BASE_NAMESPACE_DIR);
snapshotDir = new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME);
Expand All @@ -682,6 +665,14 @@ Path getDataDir() {
return dataDir;
}

Path getMobDir() {
return mobDataDir.getParent();
}

Path getMobDataDir() {
return mobDataDir;
}

Path getTmpDir() {
return new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY);
}
Expand All @@ -702,11 +693,20 @@ Path getDataNsDir(String namespace) {
return new Path(dataDir, namespace);
}

Path getTableDir(TableName tableName) {
Path getMobDataNsDir(String namespace) {
return new Path(mobDataDir, namespace);
}

Path getDataTableDir(TableName tableName) {
return new Path(getDataNsDir(tableName.getNamespaceAsString()),
tableName.getQualifierAsString());
}

Path getMobTableDir(TableName tableName) {
return new Path(getMobDataNsDir(tableName.getNamespaceAsString()),
tableName.getQualifierAsString());
}

Path getArchiveNsDir(String namespace) {
return new Path(archiveDataDir, namespace);
}
Expand Down
30 changes: 30 additions & 0 deletions ...e-server/src/test/java/org/apache/hadoop/hbase/security/access/TestHDFSAclController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,23 @@ public void testDeleteNamespace() throws Exception {
TestHDFSAclHelper.canUserScanSnapshot(TEST_UTIL, grantUser, snapshot, 6);
}
}

@Test
public void testGrantMobTable() throws Exception {
final String grantUserName = name.getMethodName();
User grantUser = User.createUserForTesting(conf, grantUserName, new String[] {});

String namespace = name.getMethodName();
TableName table = TableName.valueOf(namespace, "t1");
String snapshot = namespace + "t1";

try (Table t = TestHDFSAclHelper.createMobTable(TEST_UTIL, table)) {
TestHDFSAclHelper.put(t);
admin.snapshot(snapshot, table);
TestHDFSAclHelper.grantOnTable(TEST_UTIL, grantUserName, table, READ);
TestHDFSAclHelper.canUserScanSnapshot(TEST_UTIL, grantUser, snapshot, 6);
}
}
}

final class TestHDFSAclHelper {
Expand Down Expand Up @@ -557,6 +574,19 @@ static Table createTable(HBaseTestingUtility util, TableName tableName) throws I
return util.createTable(td, splits);
}

static Table createMobTable(HBaseTestingUtility util, TableName tableName) throws IOException {
createNamespace(util, tableName.getNamespaceAsString());
TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName)
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(COLUMN1).setMobEnabled(true)
.setMobThreshold(0).build())
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(COLUMN2).setMobEnabled(true)
.setMobThreshold(0).build())
.setOwner(User.createUserForTesting(util.getConfiguration(), "owner", new String[] {}))
.build();
byte[][] splits = new byte[][] { Bytes.toBytes("2"), Bytes.toBytes("4") };
return util.createTable(td, splits);
}

static void createTableAndPut(HBaseTestingUtility util, TableName tableNam) throws IOException {
try (Table t = createTable(util, tableNam)) {
put(t);
Expand Down

0 comments on commit 19f9ce9

Please sign in to comment.