From df5785bb1c2f241c41aa417ed5823da3b4f3acfe Mon Sep 17 00:00:00 2001 From: dunwu Date: Sat, 1 Jul 2023 20:21:48 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20hbase=20=E4=BB=A3=E7=A0=81=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- codes/javadb/hbase/pom.xml | 34 +---- .../dunwu/javadb/hbase/HBaseAdminHelper.java | 78 +++++++++-- .../github/dunwu/javadb/hbase/HBaseDemo.java | 41 ------ .../dunwu/javadb/hbase/HBaseHelper.java | 25 ++-- .../javadb/hbase/HBaseAdminHelperTests.java | 87 ++++++++++++ .../dunwu/javadb/hbase/HBaseHelperTests.java | 129 ++++++++++++++++++ 6 files changed, 304 insertions(+), 90 deletions(-) delete mode 100644 codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseDemo.java create mode 100644 codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseAdminHelperTests.java create mode 100644 codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseHelperTests.java diff --git a/codes/javadb/hbase/pom.xml b/codes/javadb/hbase/pom.xml index 4c4776d..c6c4e40 100644 --- a/codes/javadb/hbase/pom.xml +++ b/codes/javadb/hbase/pom.xml @@ -23,6 +23,7 @@ org.apache.hbase hbase-client + ${hbase.version} org.apache.hadoop @@ -40,36 +41,11 @@ 1.18.22 - - junit - junit + org.springframework.boot + spring-boot-starter-test + 2.6.3 + test - - - - - - org.apache.hbase - hbase-client - ${hbase.version} - - - - - - - - - - - junit - junit - ${junit.version} - test - - - - diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseAdminHelper.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseAdminHelper.java index 9e990df..509d8e9 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseAdminHelper.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseAdminHelper.java @@ -91,10 +91,14 @@ public class HBaseAdminHelper implements Closeable { * @param namespace 命名空间 */ public void createNamespace(String namespace) throws IOException { - NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build(); - Admin admin = getAdmin(); - admin.createNamespace(nd); - admin.close(); + Admin admin = null; + try { + admin = getAdmin(); + NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build(); + admin.createNamespace(nd); + } finally { + recycle(admin); + } } /** @@ -113,16 +117,33 @@ public class HBaseAdminHelper implements Closeable { * @param force 是否强制删除 */ public void dropNamespace(String namespace, boolean force) throws IOException { - Admin admin = getAdmin(); - if (force) { - TableName[] tableNames = getAdmin().listTableNamesByNamespace(namespace); - for (TableName name : tableNames) { - admin.disableTable(name); - admin.deleteTable(name); + Admin admin = null; + try { + admin = getAdmin(); + if (force) { + TableName[] tableNames = admin.listTableNamesByNamespace(namespace); + for (TableName name : tableNames) { + admin.disableTable(name); + admin.deleteTable(name); + } } + admin.deleteNamespace(namespace); + } finally { + recycle(admin); + } + } + + /** + * 获取所有命名空间 + */ + public String[] listNamespaces() throws IOException { + Admin admin = null; + try { + admin = getAdmin(); + return admin.listNamespaces(); + } finally { + recycle(admin); } - admin.deleteNamespace(namespace); - admin.close(); } /** @@ -212,6 +233,32 @@ public class HBaseAdminHelper implements Closeable { admin.close(); } + /** + * 获取所有表 + */ + public TableName[] listTableNames() throws IOException { + Admin admin = null; + try { + admin = getAdmin(); + return admin.listTableNames(); + } finally { + recycle(admin); + } + } + + /** + * 获取指定命名空间下的所有表 + */ + public TableName[] listTableNamesByNamespace(String namespace) throws IOException { + Admin admin = null; + try { + admin = getAdmin(); + return admin.listTableNamesByNamespace(namespace); + } finally { + recycle(admin); + } + } + /** * 获取 {@link Table} 实例 * @@ -231,4 +278,11 @@ public class HBaseAdminHelper implements Closeable { return getConnection().getAdmin(); } + private void recycle(Admin admin) { + if (null == admin) { + return; + } + IoUtil.close(admin); + } + } diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseDemo.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseDemo.java deleted file mode 100644 index e9e029a..0000000 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseDemo.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.github.dunwu.javadb.hbase; - -import lombok.extern.slf4j.Slf4j; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; - -@Slf4j -public class HBaseDemo { - - public static void main(String[] args) throws Exception { - - // 请改为配置的方式 - // String zkHosts = "192.168.31.127"; - String zkHosts = "192.168.31.255"; - // 请改为配置的方式 - String zkPort = "2181"; - // 请改为配置的方式 - String namespace = "test"; - String tablename = "test"; - Configuration conf = HBaseConfiguration.create(); - conf.set("hbase.zookeeper.quorum", zkHosts); - conf.set("hbase.zookeeper.port", zkPort); - - // 创建命名空间和表 - TableName tableName = TableName.valueOf(namespace, tablename); - HBaseAdminHelper hbaseAdminHelper = HBaseAdminHelper.newInstance(conf); - hbaseAdminHelper.enableTable(tableName); - // hbaseAdminHelper.createNamespace(namespace); - // hbaseAdminHelper.createTable(tableName, "c1"); - // - // String rowKey = IdUtil.fastSimpleUUID(); - // HBaseHelper hbaseHelper = HBaseHelper.newInstance(hbaseAdminHelper.getConnection()); - // hbaseHelper.put(tableName, rowKey, "c1", "name", "jack"); - // String value = hbaseHelper.get(tableName, rowKey, "c1", "name"); - // System.out.println("value = " + value); - - hbaseAdminHelper.close(); - } - -} diff --git a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseHelper.java b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseHelper.java index 0193694..e3405b6 100644 --- a/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseHelper.java +++ b/codes/javadb/hbase/src/main/java/io/github/dunwu/javadb/hbase/HBaseHelper.java @@ -61,6 +61,11 @@ public class HBaseHelper implements Closeable { this.connection = ConnectionFactory.createConnection(configuration); } + protected HBaseHelper(Connection connection) { + this.configuration = connection.getConfiguration(); + this.connection = connection; + } + public static synchronized HBaseHelper newInstance(Configuration configuration) throws IOException { if (configuration == null) { throw new IllegalArgumentException("configuration can not be null!"); @@ -68,6 +73,13 @@ public class HBaseHelper implements Closeable { return new HBaseHelper(configuration); } + public synchronized static HBaseHelper newInstance(Connection connection) throws IOException { + if (connection == null) { + throw new IllegalArgumentException("connection can not be null!"); + } + return new HBaseHelper(connection); + } + /** * 关闭内部持有的 HBase Connection 实例 */ @@ -555,8 +567,7 @@ public class HBaseHelper implements Closeable { * @return 一级 Map 的 key 是 Row Key；二级 Map 的 key 是列，value 是列值 */ public Map> scanFamilyMap(TableName tableName, String family, - Collection columns) - throws Exception { + Collection columns) throws Exception { HBaseFamilyRequest request = new HBaseFamilyRequest(); request.setFamily(family) .setColumns(columns) @@ -597,8 +608,7 @@ public class HBaseHelper implements Closeable { * @return 一级 Map 的 key 是 Row Key；二级 Map 的 key 是列，value 是列值 */ public Map> scanFamilyMap(TableName tableName, String family, - Collection columns, - Filter filter) throws Exception { + Collection columns, Filter filter) throws Exception { HBaseFamilyRequest request = new HBaseFamilyRequest(); request.setFamily(family) .setColumns(columns) @@ -646,8 +656,7 @@ public class HBaseHelper implements Closeable { * @return 一级 Map 的 key 是 Row Key；二级 Map 的 key 是列，value 是列值 */ public Map> scanFamilyMap(TableName tableName, String family, - Collection columns, - long minStamp, long maxStamp, Filter filter) throws Exception { + Collection columns, long minStamp, long maxStamp, Filter filter) throws Exception { HBaseFamilyRequest request = new HBaseFamilyRequest(); request.setFamily(family) .setColumns(columns) @@ -829,8 +838,8 @@ public class HBaseHelper implements Closeable { request.getPageNo(), request.getPageSize(), request.toScan()); } - public PageData pageRowData(TableName tableName, - Map> familyColumns, Integer pageNo, Integer pageSize, Scan scan) throws Exception { + public PageData pageRowData(TableName tableName, Map> familyColumns, + Integer pageNo, Integer pageSize, Scan scan) throws Exception { Table table = getTable(tableName); Map>> rowMap = new HashMap<>(); diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseAdminHelperTests.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseAdminHelperTests.java new file mode 100644 index 0000000..f665b6b --- /dev/null +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseAdminHelperTests.java @@ -0,0 +1,87 @@ +package io.github.dunwu.javadb.hbase; + +import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.ArrayUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.stream.Stream; + +/** + * HBase {@link Admin} API 测试例 + *

+ * HBaseAdminHelper 是针对 {@link Admin} 常用 API 的封装工具类 + */ +public class HBaseAdminHelperTests { + + private static HBaseAdminHelper hbaseAdminHelper = null; + + @BeforeAll + public static void init() throws IOException { + Configuration conf = HBaseConfiguration.create(); + conf.set("hbase.zookeeper.quorum", "localhost"); + conf.set("hbase.zookeeper.port", "2181"); + hbaseAdminHelper = HBaseAdminHelper.newInstance(conf); + } + + @AfterAll + public static void destroy() { + IoUtil.close(hbaseAdminHelper); + } + + @Test + @DisplayName("创建、删除、查看命名空间") + public void testNamespace() throws IOException { + // 创建命名空间 + hbaseAdminHelper.createNamespace("temp"); + dumpNamespaces(); + // 删除命名空间 + hbaseAdminHelper.dropNamespace("temp", true); + dumpNamespaces(); + } + + private void dumpNamespaces() throws IOException { + String[] namespaces = hbaseAdminHelper.listNamespaces(); + System.out.println("命名空间："); + if (ArrayUtil.isNotEmpty(namespaces)) { + Stream.of(namespaces).forEach(System.out::println); + } + } + + @Test + @DisplayName("创建、删除、启用、禁用查看表") + public void testTable() throws IOException { + // 创建命名空间 + hbaseAdminHelper.createNamespace("temp"); + // 创建名为 test 的表，并含有两个列族 d 和 b + hbaseAdminHelper.createTable(TableName.valueOf("temp:test"), "d", "b"); + // 查看表 + dumpTablesInNamespace("temp"); + // 禁用表 + hbaseAdminHelper.disableTable(TableName.valueOf("temp:test")); + // 启用表 + hbaseAdminHelper.enableTable(TableName.valueOf("temp:test")); + // 删除表 + hbaseAdminHelper.dropTable(TableName.valueOf("temp:test")); + // 查看表 + dumpTablesInNamespace("temp"); + // 删除命名空间 + hbaseAdminHelper.dropNamespace("temp", true); + } + + private void dumpTablesInNamespace(String namespace) throws IOException { + TableName[] tableNames = hbaseAdminHelper.listTableNamesByNamespace(namespace); + System.out.println("表："); + if (ArrayUtil.isNotEmpty(tableNames)) { + Stream.of(tableNames).forEach(System.out::println); + } + } + +} diff --git a/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseHelperTests.java b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseHelperTests.java new file mode 100644 index 0000000..c9f4afe --- /dev/null +++ b/codes/javadb/hbase/src/test/java/io/github/dunwu/javadb/hbase/HBaseHelperTests.java @@ -0,0 +1,129 @@ +package io.github.dunwu.javadb.hbase; + +import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.IdUtil; +import cn.hutool.json.JSONUtil; +import io.github.dunwu.javadb.hbase.entity.HBaseFamilyRequest; +import io.github.dunwu.javadb.hbase.entity.HBaseMultiFamilyRequest; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * HBase {@link Table} API 测试例 + *

+ * {@link HBaseHelper} 是针对 {@link Table} 常用 API 的封装工具类 + */ +public class HBaseHelperTests { + + private static HBaseHelper hbaseHelper = null; + private static HBaseAdminHelper hbaseAdminHelper = null; + + public static final String TABLE_NAME = "test:log"; + + @BeforeAll + public static void init() throws IOException { + + Configuration conf = HBaseConfiguration.create(); + conf.set("hbase.zookeeper.quorum", "localhost"); + conf.set("hbase.zookeeper.port", "2181"); + hbaseHelper = HBaseHelper.newInstance(conf); + hbaseAdminHelper = HBaseAdminHelper.newInstance(conf); + + // 创建命名空间 + hbaseAdminHelper.createNamespace("test"); + // 创建名为 test 的表，并含有两个列族 d 和 b + hbaseAdminHelper.createTable(TableName.valueOf(TABLE_NAME), "d", "b"); + } + + @AfterAll + public static void destroy() throws IOException { + hbaseAdminHelper.dropTable(TableName.valueOf(TABLE_NAME)); + hbaseAdminHelper.dropNamespace("test", true); + IoUtil.close(hbaseAdminHelper); + IoUtil.close(hbaseHelper); + } + + @Test + public void test() throws Exception { + String uuid = IdUtil.fastUUID(); + CommonInfo commonInfo = new CommonInfo(this.getClass().getCanonicalName(), "test", 100); + LogInfo logInfo = new LogInfo("INFO", "hello world"); + hbaseHelper.put(TableName.valueOf(TABLE_NAME), uuid, "d", System.currentTimeMillis(), commonInfo); + hbaseHelper.put(TableName.valueOf(TABLE_NAME), uuid, "b", System.currentTimeMillis(), logInfo); + + // 查单列的值 + String msg = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "b", "msg"); + String level = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "b", "level"); + Assertions.assertThat(msg).isEqualTo("hello world"); + Assertions.assertThat(level).isEqualTo("INFO"); + String className = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "d", "className"); + String methodName = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "d", "methodName"); + String lineNum = hbaseHelper.getColumn(TableName.valueOf(TABLE_NAME), uuid, "d", "lineNum"); + Assertions.assertThat(className).isEqualTo(this.getClass().getCanonicalName()); + Assertions.assertThat(methodName).isEqualTo("test"); + Assertions.assertThat(lineNum).isEqualTo("100"); + + // 查单列族数据 + Map familyB = hbaseHelper.getFamilyMap(TableName.valueOf(TABLE_NAME), uuid, "b"); + Map familyD = hbaseHelper.getFamilyMap(TableName.valueOf(TABLE_NAME), uuid, "d"); + Assertions.assertThat(familyB).isNotEmpty(); + Assertions.assertThat(familyD).isNotEmpty(); + System.out.println("family b" + JSONUtil.toJsonStr(familyB)); + System.out.println("family d" + JSONUtil.toJsonStr(familyD)); + + // 查多列族数据 + Map> familyColumns = new HashMap<>(2); + familyColumns.put("d", Arrays.asList("className", "methodName", "lineNum")); + familyColumns.put("b", Arrays.asList("msg", "level")); + Map> multiFamilyMap = + hbaseHelper.getMultiFamilyMap(TableName.valueOf(TABLE_NAME), uuid, familyColumns); + System.out.println("multiFamilyMap" + JSONUtil.toJsonStr(multiFamilyMap)); + + HBaseMultiFamilyRequest request = new HBaseMultiFamilyRequest(); + request.setTableName(TABLE_NAME); + request.setStartRow(uuid); + request.getFamilyColumns().put("d", familyColumns.get("d")); + request.getFamilyColumns().put("b", familyColumns.get("b")); + Map>> rowFamilyMap = hbaseHelper.scanMultiFamilyMap(request); + System.out.println("rowFamilyMap" + JSONUtil.toJsonStr(rowFamilyMap)); + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class CommonInfo { + + private String className; + private String methodName; + private int lineNum; + + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class LogInfo { + + private String level; + private String msg; + + } + +}