From 44ddf7a52b221e6085f1ba8f43e854965b5554fe Mon Sep 17 00:00:00 2001 From: dunwu Date: Thu, 26 May 2022 12:30:35 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=20Redis=20=E7=A4=BA?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- codes/javadb/javadb-redis/pom.xml | 5 + .../{ => redis}/RedissonStandaloneTest.java | 2 +- .../{ => redis/jedis}/JedisDemoTest.java | 13 +- .../{ => redis/jedis}/JedisPoolDemoTest.java | 2 +- .../javadb/redis/jedis/rank/RankDemo.java | 617 ++++++++++++++++++ .../redis/jedis/rank/RankDemoTests.java | 151 +++++ .../javadb/redis/jedis/rank/RankElement.java | 25 + .../javadb/redis/jedis/rank/RankRegion.java | 38 ++ .../redis/jedis/rank/RankRegionElement.java | 31 + codes/redis/redis-in-action/pom.xml | 6 + .../io/github/dunwu/db/redis/Chapter02.java | 28 + .../github/dunwu/db/redis/SortedSetDemo.java | 63 ++ 12 files changed, 972 insertions(+), 9 deletions(-) rename codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/{ => redis}/RedissonStandaloneTest.java (95%) rename codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/{ => redis/jedis}/JedisDemoTest.java (93%) rename codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/{ => redis/jedis}/JedisPoolDemoTest.java (97%) create mode 100644 codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemo.java create mode 100644 codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemoTests.java create mode 100644 codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankElement.java create mode 100644 codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegion.java create mode 100644 codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegionElement.java create mode 100644 codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/SortedSetDemo.java diff --git a/codes/javadb/javadb-redis/pom.xml b/codes/javadb/javadb-redis/pom.xml index 3b251ad..b58fc4a 100644 --- a/codes/javadb/javadb-redis/pom.xml +++ b/codes/javadb/javadb-redis/pom.xml @@ -33,6 +33,11 @@ test + + cn.hutool + hutool-all + 5.5.9 + org.projectlombok lombok diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/RedissonStandaloneTest.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java similarity index 95% rename from codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/RedissonStandaloneTest.java rename to codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java index 6863682..cf04e72 100644 --- a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/RedissonStandaloneTest.java +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/RedissonStandaloneTest.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb; +package io.github.dunwu.javadb.redis; import org.junit.jupiter.api.Test; import org.redisson.api.RBucket; diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/JedisDemoTest.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisDemoTest.java similarity index 93% rename from codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/JedisDemoTest.java rename to codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisDemoTest.java index 7d3a0cd..1812667 100644 --- a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/JedisDemoTest.java +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisDemoTest.java @@ -1,11 +1,10 @@ -package io.github.dunwu.javadb; +package io.github.dunwu.javadb.redis.jedis; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.exceptions.JedisConnectionException; @@ -15,9 +14,11 @@ import java.util.Set; /** * Jedis 测试例 + * * @author Zhang Peng * @see https://github.com/xetorthio/jedis */ +@Slf4j public class JedisDemoTest { private static final String REDIS_HOST = "localhost"; @@ -26,8 +27,6 @@ public class JedisDemoTest { private static Jedis jedis = null; - private static Logger logger = LoggerFactory.getLogger(JedisDemoTest.class); - @BeforeAll public static void beforeClass() { // Jedis 有多种构造方法，这里选用最简单的一种情况 @@ -36,7 +35,7 @@ public class JedisDemoTest { // 触发 ping 命令 try { jedis.ping(); - logger.debug("jedis 连接成功。"); + log.debug("jedis 连接成功。"); } catch (JedisConnectionException e) { e.printStackTrace(); } @@ -46,7 +45,7 @@ public class JedisDemoTest { public static void afterClass() { if (null != jedis) { jedis.close(); - logger.debug("jedis 关闭连接。"); + log.debug("jedis 关闭连接。"); } } diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/JedisPoolDemoTest.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java similarity index 97% rename from codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/JedisPoolDemoTest.java rename to codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java index 193edfc..d39ea12 100644 --- a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/JedisPoolDemoTest.java +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/JedisPoolDemoTest.java @@ -1,4 +1,4 @@ -package io.github.dunwu.javadb; +package io.github.dunwu.javadb.redis.jedis; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemo.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemo.java new file mode 100644 index 0000000..4110d1a --- /dev/null +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemo.java @@ -0,0 +1,617 @@ +package io.github.dunwu.javadb.redis.jedis.rank; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.collection.CollectionUtil; +import lombok.extern.slf4j.Slf4j; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Tuple; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * 利用 sorted set 实现排行榜示例 + * + * @author Zhang Peng + * @date 2022-05-26 + */ +@Slf4j +public class RankDemo { + + public static final boolean isRegionRankEnabled = true; + private final Jedis jedis; + + public RankDemo(Jedis jedis) { + this.jedis = jedis; + } + + // ================================================================================ + // 排行榜公共常量、方法 + // ================================================================================ + + /** + * 第一名 + */ + static final int FIRST = 0; + /** + * 头部排行榜长度 + */ + static final int HEAD_RANK_LENGTH = 200; + /** + * 总排行榜长度 + */ + static final long TOTAL_RANK_LENGTH = 1000; + /** + * 排行榜第一个分区长度 + */ + static final int FIRST_REGION_LEN = 1; + /** + * 普通分区长度 + */ + static final int COMMON_REGION_LEN = 50; + /** + * 排行榜最后一名位置 + */ + static final long RANK_END_OFFSET = -TOTAL_RANK_LENGTH - 1; + + /** + * 根据 member，查询成员在排行榜中的排名，从 0 开始计数 + *

+ * 如果成员不在排行榜，则统一返回 {@link #TOTAL_RANK_LENGTH} + * + * @param member zset 成员 + * @return / + */ + public RankElement getRankByMember(String member) { + if (isRegionRankEnabled) { + RankRegionElement element = getRankByMemberWithRegions(member); + return BeanUtil.toBean(element, RankElement.class); + } else { + // 排行榜采用不分区方案 + return getRankByMemberWithNoRegions(member); + } + } + + /** + * 根据从总排名的范围获取元素列表 + * + * @param begin 总排名中的起始位置 + * @param end 总排名中的结束位置 + * @param isAsc true：从低到高 / false：从高到低 + * @return / + */ + public List getRankElementList(long begin, long end, boolean isAsc) { + + if (begin < 0 || end >= TOTAL_RANK_LENGTH) { + log.error("【排行榜】请求范围 begin = {}, end = {} 超出排行榜实际范围", begin, end); + return null; + } + + if (isRegionRankEnabled) { + // 排行榜采用分区方案 + List elementList = getRankElementListWithRegions(begin, end, isAsc); + if (CollectionUtil.isEmpty(elementList)) { + return null; + } + return elementList.stream().map(i -> BeanUtil.toBean(i, RankElement.class)).collect(Collectors.toList()); + } else { + // 排行榜采用不分区方案 + return getRankElementListWithNoRegions(begin, end, isAsc); + } + } + + /** + * 更新排行榜 + * + * @param member 榜单成员 + * @param score 榜单成员分值 + */ + public void saveRank(String member, double score) { + if (isRegionRankEnabled) { + // 排行榜采用分区方案 + saveRankWithRegions(member, score); + } else { + // 排行榜采用不分区方案 + saveRankWithNoRegions(member, score); + } + } + + + // ================================================================================ + // 排行榜【不分区】方案 + // ================================================================================ + + /** + * 排行榜缓存前缀 + */ + static final String RANK = "rank"; + + /** + * 根据 member，查询成员在排行榜中的排名，从 0 开始计数 + *

+ * 如果成员不在排行榜，则统一返回 {@link #TOTAL_RANK_LENGTH} + * + * @param member zset 成员 + * @return / + */ + public RankElement getRankByMemberWithNoRegions(String member) { + Long rank = jedis.zrevrank(RANK, member); + if (rank != null) { + Set tuples = jedis.zrevrangeWithScores(RANK, rank, rank); + for (Tuple tuple : tuples) { + if (tuple.getElement().equals(member)) { + return new RankElement(member, tuple.getScore(), rank); + } + } + } + return new RankElement(member, null, TOTAL_RANK_LENGTH); + } + + /** + * 根据从总排名的范围获取元素列表 + * + * @param begin 总排名中的起始位置 + * @param end 总排名中的结束位置 + * @param isAsc true：从低到高 / false：从高到低 + * @return / + */ + private List getRankElementListWithNoRegions(long begin, long end, boolean isAsc) { + Set tuples; + if (isAsc) { + tuples = jedis.zrevrangeWithScores(RANK, begin, end); + } else { + tuples = jedis.zrangeWithScores(RANK, begin, end); + } + + if (CollectionUtil.isEmpty(tuples)) { + return null; + } + + long rank = 0; + List list = new ArrayList<>(); + for (Tuple tuple : tuples) { + RankElement elementVo = new RankElement(tuple.getElement(), tuple.getScore(), rank++); + list.add(elementVo); + } + return list; + } + + /** + * 更新【不分区】排行榜 + * + * @param member 榜单成员 + * @param score 榜单成员分值 + */ + private void saveRankWithNoRegions(final String member, final double score) { + Pipeline pipeline = jedis.pipelined(); + pipeline.zadd(RANK, score, member); + pipeline.zremrangeByRank(RANK, 0, RANK_END_OFFSET); + pipeline.sync(); + } + + + // ================================================================================ + // 排行榜【分区】方案 + // ================================================================================ + + /** + * 排行榜缓存前缀 + */ + static final String RANK_PREFIX = "rank:"; + /** + * 排行榜所有分区的分区号（分区号实际上就是该分区排名第一元素的实际排名） + */ + static final List REGIONS = getAllRankRegions(); + + /** + * 根据 member，查询成员在排行榜中的排名，从 0 开始计数 + *

+ * 如果成员不在排行榜，则统一返回 {@link #TOTAL_RANK_LENGTH} + * + * @param member zset 成员 + * @return / + */ + public RankRegionElement getRankByMemberWithRegions(String member) { + long totalRank = TOTAL_RANK_LENGTH; + for (RankRegion region : REGIONS) { + // 计算排行榜分区的 Redis Key + Long rank = jedis.zrevrank(region.getRegionKey(), member); + + if (rank != null) { + totalRank = getTotalRank(region.getRegionNo(), rank); + return new RankRegionElement(region.getRegionNo(), region.getRegionKey(), member, null, rank, + totalRank); + } + } + int lastRegionNo = getLastRegionNo(); + return new RankRegionElement(lastRegionNo, getRankRedisKey(lastRegionNo), member, null, null, totalRank); + } + + /** + * 根据从总排名的范围获取元素列表 + * + * @param begin 总排名中的起始位置 + * @param end 总排名中的结束位置 + * @param isAsc true：从低到高 / false：从高到低 + * @return / + */ + public List getRankElementListWithRegions(long begin, long end, boolean isAsc) { + if (begin < 0 || end >= TOTAL_RANK_LENGTH) { + log.error("【排行榜】请求范围 begin = {}, end = {} 超出排行榜实际范围", begin, end); + return null; + } + + List finalList = new LinkedList<>(); + for (RankRegion region : REGIONS) { + + long regionBegin = region.getRegionNo(); + long regionEnd = region.getRegionNo() + region.getMaxSize() - 1; + + if (regionBegin > end) { + break; + } + + if (regionEnd < begin) { + continue; + } + + long first = Math.max(regionBegin, begin); + long last = Math.min(regionEnd, end); + RankRegionElement firstElement = getRegionRank(first); + RankRegionElement lastElement = getRegionRank(last); + List list = getRankElementListInRegion(region, firstElement.getRank(), + lastElement.getRank(), isAsc); + if (CollectionUtil.isNotEmpty(list)) { + finalList.addAll(list); + } + } + return finalList; + } + + /** + * 获取指定分区中指定排名范围的信息 + * + * @param region 指定榜单分区 + * @param begin 起始排名 + * @param end 结束排名 + * @param isAsc true：从低到高 / false：从高到低 + * @return 匹配排名的信息 + */ + private List getRankElementListInRegion(RankRegion region, long begin, long end, boolean isAsc) { + Set tuples; + if (isAsc) { + // 从低到高排名 + tuples = jedis.zrangeWithScores(region.getRegionKey(), begin, end); + } else { + // 从高到低排名 + tuples = jedis.zrevrangeWithScores(region.getRegionKey(), begin, end); + } + + if (CollectionUtil.isEmpty(tuples)) { + return null; + } + + long regionRank = 0; + List list = new ArrayList<>(); + for (Tuple tuple : tuples) { + long totalRank = getTotalRank(region.getRegionNo(), regionRank); + RankRegionElement rankElementVo = new RankRegionElement(region.getRegionNo(), region.getRegionKey(), + tuple.getElement(), tuple.getScore(), regionRank, + totalRank); + list.add(rankElementVo); + regionRank++; + } + return list; + } + + /** + * 获取指定分区中指定排名的信息 + * + * @param region 指定榜单分区 + * @param rank 分区中的排名 + * @param isAsc true：从低到高 / false：从高到低 + * @return 匹配排名的信息 + */ + private RankRegionElement getRankElementInRegion(RankRegion region, long rank, boolean isAsc) { + Set tuples; + if (isAsc) { + // 从低到高排名 + tuples = jedis.zrangeWithScores(region.getRegionKey(), rank, rank); + } else { + // 从高到低排名 + tuples = jedis.zrevrangeWithScores(region.getRegionKey(), rank, rank); + } + + if (CollectionUtil.isEmpty(tuples)) { + return null; + } + + Tuple tuple = tuples.iterator().next(); + if (tuple == null) { + return null; + } + + long regionRank = rank; + if (isAsc) { + regionRank = region.getMaxSize() - 1; + } + + long totalRank = getTotalRank(region.getRegionNo(), rank); + return new RankRegionElement(region.getRegionNo(), region.getRegionKey(), tuple.getElement(), tuple.getScore(), + regionRank, totalRank); + } + + /** + * 获取最后一名 + */ + private RankRegionElement getMinRankElementInRegion(RankRegion region) { + return getRankElementInRegion(region, FIRST, true); + } + + /** + * 获取第一名 + */ + private RankRegionElement getMaxRankElementInRegion(RankRegion region) { + return getRankElementInRegion(region, FIRST, false); + } + + /** + * 更新【分区】排行榜 + * + * @param member 榜单成员 + * @param score 榜单成员分值 + */ + public void saveRankWithRegions(final String member, final double score) { + + List regions = new LinkedList<>(REGIONS); + + // member 的原始排名 + RankRegionElement oldRank = null; + for (RankRegion region : regions) { + + region.setSize(jedis.zcard(region.getRegionKey())); + region.setMin(getMinRankElementInRegion(region)); + region.setMax(getMaxRankElementInRegion(region)); + + // 查找 member 是否已经在榜单中 + Long rank = jedis.zrevrank(region.getRegionKey(), member); + if (rank != null) { + jedis.zrevrangeWithScores(region.getRegionKey(), rank, rank); + oldRank = getRankElementInRegion(region, rank, false); + } + } + + Pipeline pipeline = jedis.pipelined(); + // 如果成员已入榜，并且无任何变化，无需任何修改 + if (oldRank != null) { + if (oldRank.getMember().equals(member) && oldRank.getScore() == score) { + log.info("【排行榜】member = {}, score = {} 值没有变化，无需任何修改", member, score); + return; + } + + // 成员已经在 10W 排行榜中，先将旧记录自适应删除 + if (oldRank.getTotalRank() < TOTAL_RANK_LENGTH) { + log.info("【排行榜】member = {} 已入 TOP {}，rank = {}", member, TOTAL_RANK_LENGTH, oldRank); + // 先将原始排名记录删除，并动态调整所有分区 + deleteWithAutoAdjust(oldRank, regions, pipeline); + } + } + + // 将成员的记录插入到合适的分区中，并自适应调整各分区 + addWithAutoAdjust(member, score, regions, pipeline); + pipeline.syncAndReturnAll(); + + long newRank = TOTAL_RANK_LENGTH; + for (RankRegion region : regions) { + Long rank = jedis.zrevrank(region.getRegionKey(), member); + if (rank != null) { + newRank = getTotalRank(region.getRegionNo(), rank); + break; + } + } + log.info("【排行榜】member = {}, score = {}, 排名：{}", member, score, newRank); + + if (oldRank != null && oldRank.getTotalRank() < HEAD_RANK_LENGTH && newRank >= HEAD_RANK_LENGTH) { + log.info("【排行榜】member = {} 跌出 TOP {}，oldRank = {}, newRank = {}", member, HEAD_RANK_LENGTH, oldRank, + newRank); + } + } + + /** + * 根据 member，score 将成员的记录插入到合适的分区中，如果没有合适的分区，说明在 10W 名以外，则不插入 + *

+ * 如果成员在 {@link #TOTAL_RANK_LENGTH} 以内排行榜，则返回真实排名；否则，则统一返回 {@link #TOTAL_RANK_LENGTH} + * + * @param member zset 成员 + * @param score 成员分值 + */ + private void addWithAutoAdjust(String member, double score, List regions, Pipeline pipeline) { + + String insertedMember = member; + double insertedScore = score; + + for (RankRegion region : regions) { + + // 判断分区长度 + if (region.getSize() < region.getMaxSize()) { + // 如果分区中实际数据量小于分区最大长度，则直接将成员插入排行榜即可： + // 由于排行榜是按照分值从高到低排序，各分区也是有序排列。 + // 分区没有满的情况下，不会创建新的分区，所以，此时必然是最后一个分区。 + pipeline.zadd(region.getRegionKey(), insertedScore, insertedMember); + region.setSize(region.getSize() + 1); + break; + } + + // 当前分区不为空，取最后一名 + if (region.getMin() == null) { + log.error("【排行榜】【删除老记录】key = {} 未找到最后一名数据！", region.getRegionKey()); + break; + } + + // 待插入分值比分区最小值还小 + if (region.getMin().getScore() >= insertedScore) { + continue; + } + + // 待插入分值大于当前分区的最小值，当前分区即为合适插入的分区 + // 将待插入成员、分值写入 + pipeline.zadd(region.getRegionKey(), insertedScore, insertedMember); + + // 从本分区中移出最后一名 + pipeline.zrem(region.getRegionKey(), region.getMin().getMember()); + + // 移入下一个分区 + insertedMember = region.getMin().getMember(); + insertedScore = region.getMin().getScore(); + } + } + + /** + * 先将原始排名记录从所属分区中删除，并动态调整之后的分区 + */ + private void deleteWithAutoAdjust(RankRegionElement oldRank, List regions, Pipeline pipeline) { + + // 计算排行榜分区的 Redis Key + pipeline.zrem(oldRank.getRegionKey(), oldRank.getMember()); + log.info("【排行榜】【删除老记录】删除原始记录：key = {}, member = {}", oldRank.getRegionKey(), oldRank.getMember()); + + int prevRegionNo = oldRank.getRegionNo(); + RankRegion prevRegion = null; + for (RankRegion region : regions) { + + // prevRegion 及之前的分区无需处理 + if (Objects.equals(region.getRegionNo(), prevRegionNo)) { + prevRegion = region; + continue; + } + if (region.getRegionNo() < oldRank.getRegionNo()) { continue; } + + // 当前分区如果为空，则无需调整，结束 + if (region.getSize() == null || region.getSize() == 0L) { + log.info("【排行榜】【删除老记录】key = {} 数据为空，无需处理", region.getRegionKey()); + break; + } + + // 当前分区不为空，取第一名 + if (region.getMax() == null) { + log.error("【排行榜】【删除老记录】key = {} 未找到第一名数据！", region.getRegionKey()); + break; + } + + if (prevRegion == null) { + break; + } + + // 从本分区中移出第一名 + pipeline.zrem(region.getRegionKey(), region.getMax().getMember()); + region.setSize(region.getSize() - 1); + // 移入上一个分区 + pipeline.zadd(prevRegion.getRegionKey(), region.getMax().getScore(), region.getMax().getMember()); + prevRegion.setSize(prevRegion.getSize() + 1); + // 替换上一分区 key + prevRegion = region; + } + } + + /** + * 获取排行榜所有分区 + *

+ * 排行榜存储 10W 条数据，分区规则为： + * 第一个分区，以 0 开始，存储 100 条数据（因为 TOP 100 查询频率高，所以分区大小设小一点，提高查询速度） + * 最后一个分区，以 95100 开始，存储 4900 条数据； + * 其他分区，都存储 5000 条数据 + */ + private static List getAllRankRegions() { + List regions = new ArrayList<>(); + RankRegion firstRegion = new RankRegion(FIRST, getRankRedisKey(FIRST), null, getRegionLength(FIRST)); + regions.add(firstRegion); + for (int index = FIRST_REGION_LEN; index < TOTAL_RANK_LENGTH; index = index + COMMON_REGION_LEN) { + RankRegion region = new RankRegion(index, getRankRedisKey(index), null, getRegionLength(index)); + regions.add(region); + } + return regions; + } + + /** + * 根据排行榜每个分区的第一个索引数字，获取该分区的长度 + *

+ * 分区大小的规则： + * 第一个分区，以 0 开始，存储 100 条数据； + * 最后一个分区，以 95100 开始，存储 4900 条数据； + * 其他分区，都存储 5000 条数据 + * + * @param region 分区第一条数据的索引 + * @return 分区的长度 + */ + private static long getRegionLength(int region) { + final int LAST = (int) ((TOTAL_RANK_LENGTH - 1) / COMMON_REGION_LEN * COMMON_REGION_LEN + FIRST_REGION_LEN); + switch (region) { + case FIRST: + return FIRST_REGION_LEN; + case LAST: + return COMMON_REGION_LEN - FIRST_REGION_LEN; + default: + return COMMON_REGION_LEN; + } + } + + /** + * 根据分区和分区中的排名，返回总排名 + */ + private static long getTotalRank(long regionNo, long rank) { + for (RankRegion region : REGIONS) { + if (region.getRegionNo().longValue() == regionNo) { + return regionNo + rank; + } + } + // 如果分区不存在，则统一返回 TOTAL_RANK_LENGTH + return TOTAL_RANK_LENGTH; + } + + /** + * 根据总排名，返回该排名应该所属的分区及分区中的排名信息 + */ + private static RankRegionElement getRegionRank(long totalRank) { + + if (totalRank < 0 || totalRank >= TOTAL_RANK_LENGTH) { return null; } + + long length = totalRank; + for (RankRegion region : REGIONS) { + if (region.getMaxSize() > length) { + return new RankRegionElement(region.getRegionNo(), region.getRegionKey(), null, null, length, + totalRank); + } else { + length -= region.getMaxSize(); + } + } + return null; + } + + /** + * 根据总排名，计算得出排名所属分区 + */ + private static int getRegionByTotalRank(long totalRank) { + if (totalRank < FIRST_REGION_LEN) { + return 0; + } + return (int) (totalRank / COMMON_REGION_LEN * COMMON_REGION_LEN + FIRST_REGION_LEN); + } + + /** + * 获取最后一个分区 + */ + private static int getLastRegionNo() { + return (int) ((TOTAL_RANK_LENGTH / COMMON_REGION_LEN - 1) * COMMON_REGION_LEN + FIRST_REGION_LEN); + } + + /** + * 排行榜缓存 Key + * + * @param regionNo 该分区第一个元素的排名 + */ + private static String getRankRedisKey(long regionNo) { + return RANK_PREFIX + regionNo; + } + +} diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemoTests.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemoTests.java new file mode 100644 index 0000000..f6a434a --- /dev/null +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankDemoTests.java @@ -0,0 +1,151 @@ +package io.github.dunwu.javadb.redis.jedis.rank; + +import cn.hutool.core.util.RandomUtil; +import cn.hutool.core.util.StrUtil; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.*; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Tuple; +import redis.clients.jedis.exceptions.JedisConnectionException; + +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +/** + * 测试 {@link RankDemo} + * + * @author Zhang Peng + * @date 2022-05-24 + */ +@Slf4j +@DisplayName("使用 zset 维护分区的排行榜缓存") +public class RankDemoTests { + + private static final String REDIS_HOST = "localhost"; + private static final int REDIS_PORT = 6379; + private static Jedis jedis = null; + private RankDemo rank; + + @BeforeAll + public static void beforeClass() { + // Jedis 有多种构造方法，这里选用最简单的一种情况 + jedis = new Jedis(REDIS_HOST, REDIS_PORT); + + // 触发 ping 命令 + try { + jedis.ping(); + jedis.select(0); + log.debug("jedis 连接成功。"); + } catch (JedisConnectionException e) { + e.printStackTrace(); + } + } + + @AfterAll + public static void afterClass() { + if (null != jedis) { + jedis.close(); + log.debug("jedis 关闭连接。"); + } + } + + @BeforeEach + public void beforeEach() { + rank = new RankDemo(jedis); + } + + @Test + @DisplayName("刷新 MOCK 数据") + public void refreshMockData() { + log.info("刷新 MOCK 数据"); + + // 清理所有排行榜分区 + for (RankRegion region : RankDemo.REGIONS) { + jedis.del(region.getRegionKey()); + } + jedis.del(RankDemo.RANK); + + for (int i = 0; i < RankDemo.TOTAL_RANK_LENGTH; i++) { + double score = RandomUtil.randomDouble(100.0, 10000.0); + String member = StrUtil.format("id-{}", i); + rank.saveRank(member, score); + } + } + + @Test + @DisplayName("测试各分区最大值、最小值") + public void getRankElementList() { + List list = rank.getRankElementList(0, 99, false); + System.out.println(list); + Assertions.assertEquals(100, list.size()); + } + + @Test + @DisplayName("添加新纪录") + public void testAdd() { + + String member1 = StrUtil.format("id-{}", RankDemo.TOTAL_RANK_LENGTH + 1); + rank.saveRank(member1, 20000.0); + + String member2 = StrUtil.format("id-{}", RankDemo.TOTAL_RANK_LENGTH + 2); + rank.saveRank(member2, 1.0); + + RankElement rank1 = rank.getRankByMember(member1); + RankElement rank2 = rank.getRankByMember(member2); + Assertions.assertEquals(RankDemo.FIRST, rank1.getTotalRank()); + Assertions.assertEquals(RankDemo.TOTAL_RANK_LENGTH, rank2.getTotalRank()); + } + + + @Nested + @DisplayName("分区方案特殊测试") + public class RegionTest { + + @Test + @DisplayName("测试各分区长度") + public void testRegionLength() { + for (RankRegion region : RankDemo.REGIONS) { + Long size = jedis.zcard(region.getRegionKey()); + log.info("【排行榜】redisKey = {}, count = {}", region.getRegionKey(), size); + Assertions.assertEquals(region.getMaxSize(), size); + } + } + + @Test + @DisplayName("测试各分区最大值、最小值") + public void testRegionSort() { + // 按序获取每个分区的最大值、最小值 + List maxScores = new LinkedList<>(); + List minScores = new LinkedList<>(); + for (RankRegion region : RankDemo.REGIONS) { + Set minSet = jedis.zrangeWithScores(region.getRegionKey(), 0, 0); + Tuple min = minSet.iterator().next(); + minScores.add(min.getScore()); + + Set maxSet = jedis.zrevrangeWithScores(region.getRegionKey(), 0, 0); + Tuple max = maxSet.iterator().next(); + maxScores.add(max.getScore()); + } + System.out.println(maxScores); + System.out.println(minScores); + + // 最大值、最小值数量必然相同 + Assertions.assertEquals(maxScores.size(), minScores.size()); + + for (int i = 0; i < minScores.size(); i++) { + compareMinScore(maxScores, i, minScores.get(i)); + } + } + + public void compareMinScore(List maxScores, int region, double score) { + for (int i = region + 1; i < maxScores.size(); i++) { + Assertions.assertFalse(score <= maxScores.get(i), + StrUtil.format("region = {}, score = {} 的最小值小于后续分区中的数值（region = {}, score = {}）", + region, score, i, maxScores.get(i))); + } + } + + } + +} diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankElement.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankElement.java new file mode 100644 index 0000000..ec03bfa --- /dev/null +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankElement.java @@ -0,0 +1,25 @@ +package io.github.dunwu.javadb.redis.jedis.rank; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 排行榜元素信息 + * + * @author Zhang Peng + * @date 2022-05-26 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RankElement { + + /** zset member */ + private String member; + /** zset score */ + private Double score; + /** 总排名 */ + private Long totalRank; + +} diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegion.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegion.java new file mode 100644 index 0000000..e865caf --- /dev/null +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegion.java @@ -0,0 +1,38 @@ +package io.github.dunwu.javadb.redis.jedis.rank; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 排行榜分区信息实体 + * + * @author Zhang Peng + * @date 2022-05-26 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RankRegion { + + /** 排行榜分区号 */ + private Integer regionNo; + /** 排行榜分区 Redis Key */ + private String regionKey; + /** 分区实际大小 */ + private Long size; + /** 分区最大大小 */ + private Long maxSize; + /** 分区中的最小值 */ + private RankRegionElement min; + /** 分区中的最大值 */ + private RankRegionElement max; + + public RankRegion(Integer regionNo, String regionKey, Long size, Long maxSize) { + this.regionNo = regionNo; + this.regionKey = regionKey; + this.size = size; + this.maxSize = maxSize; + } + +} diff --git a/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegionElement.java b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegionElement.java new file mode 100644 index 0000000..1a52e18 --- /dev/null +++ b/codes/javadb/javadb-redis/src/test/java/io/github/dunwu/javadb/redis/jedis/rank/RankRegionElement.java @@ -0,0 +1,31 @@ +package io.github.dunwu.javadb.redis.jedis.rank; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 排行榜（分区）元素信息 + * + * @author Zhang Peng + * @date 2022-05-25 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RankRegionElement { + + /** 排行榜分区号 */ + private Integer regionNo; + /** 排行榜分区 Redis Key */ + private String regionKey; + /** zset member */ + private String member; + /** zset score */ + private Double score; + /** 当前分区的排名 */ + private Long rank; + /** 总排名 */ + private Long totalRank; + +} diff --git a/codes/redis/redis-in-action/pom.xml b/codes/redis/redis-in-action/pom.xml index 3c2c2ed..5699fbd 100644 --- a/codes/redis/redis-in-action/pom.xml +++ b/codes/redis/redis-in-action/pom.xml @@ -62,5 +62,11 @@ javatuples 1.1 + + + cn.hutool + hutool-all + 5.5.9 + diff --git a/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/Chapter02.java b/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/Chapter02.java index 936153e..b8c8652 100644 --- a/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/Chapter02.java +++ b/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/Chapter02.java @@ -451,6 +451,7 @@ public class Chapter02 { } + public static class Inventory { private String id; @@ -469,6 +470,33 @@ public class Chapter02 { return new Inventory(id); } + public String getId() { + return id; + } + + public Inventory setId(String id) { + this.id = id; + return this; + } + + public String getData() { + return data; + } + + public Inventory setData(String data) { + this.data = data; + return this; + } + + public long getTime() { + return time; + } + + public Inventory setTime(long time) { + this.time = time; + return this; + } + } } diff --git a/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/SortedSetDemo.java b/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/SortedSetDemo.java new file mode 100644 index 0000000..f8ab9f1 --- /dev/null +++ b/codes/redis/redis-in-action/src/main/java/io/github/dunwu/db/redis/SortedSetDemo.java @@ -0,0 +1,63 @@ +package io.github.dunwu.db.redis; + +import cn.hutool.core.util.RandomUtil; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.Tuple; + +import java.util.Set; + +/** + * @author Zhang Peng + * @date 2022-05-20 + */ +public class SortedSetDemo { + + public static final String TEST_KEY = "test:zset"; + public static final Jedis conn = new Jedis("localhost"); + + public static void main(String[] args) { + conn.select(0); + // zadd(conn); + zrem(conn); + // zrank(conn); + // zrange(conn); + zcard(conn); + conn.close(); + } + + public static void zadd(Jedis conn) { + for (int i = 0; i < 100; i++) { + conn.zadd(TEST_KEY, RandomUtil.randomDouble(10000.0), RandomUtil.randomString(6)); + } + conn.zadd(TEST_KEY, 20000.0, "THETOP"); + } + + public static void zrem(Jedis conn) { + int len = 10; + int end = -len - 1; + conn.zremrangeByRank(TEST_KEY, 0, end); + } + + public static void zcard(Jedis conn) { + System.out.println("count = " + conn.zcard(TEST_KEY)); + } + + public static void zrank(Jedis conn) { + System.out.println("THETOP 从低到高排名：" + conn.zrank(TEST_KEY, "THETOP")); + System.out.println("THETOP 从高到低排名：" + conn.zrevrank(TEST_KEY, "THETOP")); + } + + public static void zrange(Jedis conn) { + System.out.println("查看从低到高第 1 名：" + conn.zrange(TEST_KEY, 0, 0)); + System.out.println("查看从高到低第 1 名：" + conn.zrevrange(TEST_KEY, 0, 0)); + System.out.println("查看从高到低前 10 名：" + conn.zrevrange(TEST_KEY, 0, 9)); + Set tuples = conn.zrevrangeWithScores(TEST_KEY, 0, 0); + for (Tuple tuple : tuples) { + System.out.println(tuple.getElement()); + System.out.println(tuple.getScore()); + } + + System.out.println("查看从高到低前 10 名：" + conn.zrevrangeWithScores(TEST_KEY, 0, 0)); + } + +}