dunwu
/
db-tutorial
mirror of https://github.com/dunwu/db-tutorial.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: hbase 代码示例

dependabot/maven/codes/middleware/flyway/com.h2database-h2-2.2.220
dunwu 2023-07-01 20:21:48 +08:00
parent c82e9d737a
commit df5785bb1c
6 changed files with 304 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
codes/javadb/hbase/pom.xml
View File

@ -23,6 +23,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -40,36 +41,11 @@
<version>1.18.22</version>
</dependency>
<!-- test begin -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>2.6.3</version>
<scope>test</scope>
</dependency>
<!-- test end -->
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>io.github.dunwu</groupId>-->
<!-- <artifactId>dunwu-tool-core</artifactId>-->
<!-- <version>${dunwu.version}</version>-->
<!-- </dependency>-->
<!-- test begin -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- test end -->
</dependencies>
</dependencyManagement>
</project>

78
codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseAdminHelper.java
View File

@ -91,10 +91,14 @@ public class HBaseAdminHelper implements Closeable {
* @param namespace 命名空间
*/
public void createNamespace(String namespace) throws IOException {
NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build();
Admin admin = getAdmin();
admin.createNamespace(nd);
admin.close();
Admin admin = null;
try {
admin = getAdmin();
NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build();
admin.createNamespace(nd);
} finally {
recycle(admin);
}
}
/**
@ -113,16 +117,33 @@ public class HBaseAdminHelper implements Closeable {
* @param force 是否强制删除
*/
public void dropNamespace(String namespace, boolean force) throws IOException {
Admin admin = getAdmin();
if (force) {
TableName[] tableNames = getAdmin().listTableNamesByNamespace(namespace);
for (TableName name : tableNames) {
admin.disableTable(name);
admin.deleteTable(name);
Admin admin = null;
try {
admin = getAdmin();
if (force) {
TableName[] tableNames = admin.listTableNamesByNamespace(namespace);
for (TableName name : tableNames) {
admin.disableTable(name);
admin.deleteTable(name);
}
}
admin.deleteNamespace(namespace);
} finally {
recycle(admin);
}
}
/**
* 获取所有命名空间
*/
public String[] listNamespaces() throws IOException {
Admin admin = null;
try {
admin = getAdmin();
return admin.listNamespaces();
} finally {
recycle(admin);
}
admin.deleteNamespace(namespace);
admin.close();
}
/**
@ -212,6 +233,32 @@ public class HBaseAdminHelper implements Closeable {
admin.close();
}
/**
* 获取所有表
*/
public TableName[] listTableNames() throws IOException {
Admin admin = null;
try {
admin = getAdmin();
return admin.listTableNames();
} finally {
recycle(admin);
}
}
/**
* 获取指定命名空间下的所有表
*/
public TableName[] listTableNamesByNamespace(String namespace) throws IOException {
Admin admin = null;
try {
admin = getAdmin();
return admin.listTableNamesByNamespace(namespace);
} finally {
recycle(admin);
}
}
/**
* 获取 {@link Table} 实例
*
@ -231,4 +278,11 @@ public class HBaseAdminHelper implements Closeable {
return getConnection().getAdmin();
}
private void recycle(Admin admin) {
if (null == admin) {
return;
}
IoUtil.close(admin);
}
}

41
codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseDemo.java
View File

@ -1,41 +0,0 @@
package io.github.dunwu.javadb.hbase;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
@Slf4j
public class HBaseDemo {
public static void main(String[] args) throws Exception {
// 请改为配置的方式
// String zkHosts = "192.168.31.127";
String zkHosts = "192.168.31.255";
// 请改为配置的方式
String zkPort = "2181";
// 请改为配置的方式
String namespace = "test";
String tablename = "test";
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", zkHosts);
conf.set("hbase.zookeeper.port", zkPort);
// 创建命名空间和表
TableName tableName = TableName.valueOf(namespace, tablename);
HBaseAdminHelper hbaseAdminHelper = HBaseAdminHelper.newInstance(conf);
hbaseAdminHelper.enableTable(tableName);
// hbaseAdminHelper.createNamespace(namespace);
// hbaseAdminHelper.createTable(tableName, "c1");
//
// String rowKey = IdUtil.fastSimpleUUID();
// HBaseHelper hbaseHelper = HBaseHelper.newInstance(hbaseAdminHelper.getConnection());
// hbaseHelper.put(tableName, rowKey, "c1", "name", "jack");
// String value = hbaseHelper.get(tableName, rowKey, "c1", "name");
// System.out.println("value = " + value);
hbaseAdminHelper.close();
}
}

25
codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseHelper.java
View File

@ -61,6 +61,11 @@ public class HBaseHelper implements Closeable {
this.connection = ConnectionFactory.createConnection(configuration);
}
protected HBaseHelper(Connection connection) {
this.configuration = connection.getConfiguration();
this.connection = connection;
}
public static synchronized HBaseHelper newInstance(Configuration configuration) throws IOException {
if (configuration == null) {
throw new IllegalArgumentException("configuration can not be null!");
@ -68,6 +73,13 @@ public class HBaseHelper implements Closeable {
return new HBaseHelper(configuration);
}
public synchronized static HBaseHelper newInstance(Connection connection) throws IOException {
if (connection == null) {
throw new IllegalArgumentException("connection can not be null!");
}
return new HBaseHelper(connection);
}
/**
* 关闭内部持有的 HBase Connection 实例
*/
@ -555,8 +567,7 @@ public class HBaseHelper implements Closeable {
* @return 一级 Map 的 key 是 Row Key；二级 Map 的 key 是列，value 是列值
*/
public Map<String, Map<String, String>> scanFamilyMap(TableName tableName, String family,
Collection<String> columns)
throws Exception {
Collection<String> columns) throws Exception {
HBaseFamilyRequest request = new HBaseFamilyRequest();
request.setFamily(family)
.setColumns(columns)
@ -597,8 +608,7 @@ public class HBaseHelper implements Closeable {
* @return 一级 Map 的 key 是 Row Key；二级 Map 的 key 是列，value 是列值
*/
public Map<String, Map<String, String>> scanFamilyMap(TableName tableName, String family,
Collection<String> columns,
Filter filter) throws Exception {
Collection<String> columns, Filter filter) throws Exception {
HBaseFamilyRequest request = new HBaseFamilyRequest();
request.setFamily(family)
.setColumns(columns)
@ -646,8 +656,7 @@ public class HBaseHelper implements Closeable {
* @return 一级 Map 的 key 是 Row Key；二级 Map 的 key 是列，value 是列值
*/
public Map<String, Map<String, String>> scanFamilyMap(TableName tableName, String family,
Collection<String> columns,
long minStamp, long maxStamp, Filter filter) throws Exception {
Collection<String> columns, long minStamp, long maxStamp, Filter filter) throws Exception {
HBaseFamilyRequest request = new HBaseFamilyRequest();
request.setFamily(family)
.setColumns(columns)
@ -829,8 +838,8 @@ public class HBaseHelper implements Closeable {
request.getPageNo(), request.getPageSize(), request.toScan());
}
public PageData<HBaseRowData> pageRowData(TableName tableName,
Map<String, Collection<String>> familyColumns, Integer pageNo, Integer pageSize, Scan scan) throws Exception {
public PageData<HBaseRowData> pageRowData(TableName tableName, Map<String, Collection<String>> familyColumns,
Integer pageNo, Integer pageSize, Scan scan) throws Exception {
Table table = getTable(tableName);
Map<String, Map<String, Map<String, String>>> rowMap = new HashMap<>();

87
codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseAdminHelperTests.java 100644
View File

@ -0,0 +1,87 @@
package io.github.dunwu.javadb.hbase;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.ArrayUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.stream.Stream;
/**
* HBase {@link Admin} API 测试例
* <p>
* HBaseAdminHelper 是针对 {@link Admin} 常用 API 的封装工具类
*/
public class HBaseAdminHelperTests {
private static HBaseAdminHelper hbaseAdminHelper = null;
@BeforeAll
public static void init() throws IOException {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "localhost");
conf.set("hbase.zookeeper.port", "2181");
hbaseAdminHelper = HBaseAdminHelper.newInstance(conf);
}
@AfterAll
public static void destroy() {
IoUtil.close(hbaseAdminHelper);
}
@Test
@DisplayName("创建、删除、查看命名空间")
public void testNamespace() throws IOException {
// 创建命名空间
hbaseAdminHelper.createNamespace("temp");
dumpNamespaces();
// 删除命名空间
hbaseAdminHelper.dropNamespace("temp", true);
dumpNamespaces();
}
private void dumpNamespaces() throws IOException {
String[] namespaces = hbaseAdminHelper.listNamespaces();
System.out.println("命名空间：");
if (ArrayUtil.isNotEmpty(namespaces)) {
Stream.of(namespaces).forEach(System.out::println);
}
}
@Test
@DisplayName("创建、删除、启用、禁用查看表")
public void testTable() throws IOException {
// 创建命名空间
hbaseAdminHelper.createNamespace("temp");
// 创建名为 test 的表，并含有两个列族 d 和 b
hbaseAdminHelper.createTable(TableName.valueOf("temp:test"), "d", "b");
// 查看表
dumpTablesInNamespace("temp");
// 禁用表
hbaseAdminHelper.disableTable(TableName.valueOf("temp:test"));
// 启用表
hbaseAdminHelper.enableTable(TableName.valueOf("temp:test"));
// 删除表
hbaseAdminHelper.dropTable(TableName.valueOf("temp:test"));
// 查看表
dumpTablesInNamespace("temp");
// 删除命名空间
hbaseAdminHelper.dropNamespace("temp", true);
}
private void dumpTablesInNamespace(String namespace) throws IOException {
TableName[] tableNames = hbaseAdminHelper.listTableNamesByNamespace(namespace);
System.out.println("表：");
if (ArrayUtil.isNotEmpty(tableNames)) {
Stream.of(tableNames).forEach(System.out::println);
}
}
}

129
codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseHelperTests.java 100644
View File

@ -0,0 +1,129 @@
package io.github.dunwu.javadb.hbase;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.json.JSONUtil;
import io.github.dunwu.javadb.hbase.entity.HBaseFamilyRequest;
import io.github.dunwu.javadb.hbase.entity.HBaseMultiFamilyRequest;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Table;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
/**
* HBase {@link Table} API 测试例
* <p>
* {@link HBaseHelper} 是针对 {@link Table} 常用 API 的封装工具类
*/
public class HBaseHelperTests {
private static HBaseHelper hbaseHelper = null;
private static HBaseAdminHelper hbaseAdminHelper = null;
public static final String TABLE_NAME = "test:log";
@BeforeAll
public static void init() throws IOException {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "localhost");
conf.set("hbase.zookeeper.port", "2181");
hbaseHelper = HBaseHelper.newInstance(conf);
hbaseAdminHelper = HBaseAdminHelper.newInstance(conf);
// 创建命名空间
hbaseAdminHelper.createNamespace("test");
// 创建名为 test 的表，并含有两个列族 d 和 b
hbaseAdminHelper.createTable(TableName.valueOf(TABLE_NAME), "d", "b");
}
@AfterAll
public static void destroy() throws IOException {
hbaseAdminHelper.dropTable(TableName.valueOf(TABLE_NAME));
hbaseAdminHelper.dropNamespace("test", true);
IoUtil.close(hbaseAdminHelper);
IoUtil.close(hbaseHelper);
}
@Test
public void test() throws Exception {
String uuid = IdUtil.fastUUID();
CommonInfo commonInfo = new CommonInfo(this.getClass().getCanonicalName(), "test", 100);
LogInfo logInfo = new LogInfo("INFO", "hello world");
hbaseHelper.put(TableName.valueOf(TABLE_NAME), uuid, "d", System.currentTimeMillis(), commonInfo);
hbaseHelper.put(TableName.valueOf(TABLE_NAME), uuid, "b", System.currentTimeMillis(), logInfo);
// 查单列的值
String msg = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "b", "msg");
String level = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "b", "level");
Assertions.assertThat(msg).isEqualTo("hello world");
Assertions.assertThat(level).isEqualTo("INFO");
String className = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "d", "className");
String methodName = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "d", "methodName");
String lineNum = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "d", "lineNum");
Assertions.assertThat(className).isEqualTo(this.getClass().getCanonicalName());
Assertions.assertThat(methodName).isEqualTo("test");
Assertions.assertThat(lineNum).isEqualTo("100");
// 查单列族数据
Map<String, String> familyB = hbaseHelper.getFamilyMap(TableName.valueOf(TABLE_NAME), uuid, "b");
Map<String, String> familyD = hbaseHelper.getFamilyMap(TableName.valueOf(TABLE_NAME), uuid, "d");
Assertions.assertThat(familyB).isNotEmpty();
Assertions.assertThat(familyD).isNotEmpty();
System.out.println("family b" + JSONUtil.toJsonStr(familyB));
System.out.println("family d" + JSONUtil.toJsonStr(familyD));
// 查多列族数据
Map<String, Collection<String>> familyColumns = new HashMap<>(2);
familyColumns.put("d", Arrays.asList("className", "methodName", "lineNum"));
familyColumns.put("b", Arrays.asList("msg", "level"));
Map<String, Map<String, String>> multiFamilyMap =
hbaseHelper.getMultiFamilyMap(TableName.valueOf(TABLE_NAME), uuid, familyColumns);
System.out.println("multiFamilyMap" + JSONUtil.toJsonStr(multiFamilyMap));
HBaseMultiFamilyRequest request = new HBaseMultiFamilyRequest();
request.setTableName(TABLE_NAME);
request.setStartRow(uuid);
request.getFamilyColumns().put("d", familyColumns.get("d"));
request.getFamilyColumns().put("b", familyColumns.get("b"));
Map<String, Map<String, Map<String, String>>> rowFamilyMap = hbaseHelper.scanMultiFamilyMap(request);
System.out.println("rowFamilyMap" + JSONUtil.toJsonStr(rowFamilyMap));
}
@Data
@NoArgsConstructor
@AllArgsConstructor
static class CommonInfo {
private String className;
private String methodName;
private int lineNum;
}
@Data
@NoArgsConstructor
@AllArgsConstructor
static class LogInfo {
private String level;
private String msg;
}
}