redis7高级篇3 数据量亿级别的统计分析(hyperloglog,bitmap,geo)

一亿级别统计分类

1.1 统计分类

1.聚合统计：统计多个集合聚合的结果，也就是多个集合之间交并差的统计。
2.排序统计：在需要展示最新列表，排行榜等场景时，如果数据更新频繁或者需要分页时，建议使用zset127.0.0.1:6379> zadd pl 111222 beijing 111223 tianjing 111333 shanghai
(integer) 3
127.0.0.1:6379> zrange pl 0 1
1) "beijing"
2) "tianjing"
127.0.0.1:6379> zrangebyscore pl 111222 111233 limit 0 5
1) "beijing"
2) "tianjing"
127.0.0.1:6379> zrangebyscore pl 111222 1114444 limit 0 5
1) "beijing"
2) "tianjing"
3) "shanghai"
127.0.0.1:6379>

3.二值统计：集合中的元素只有0和1 ，钉钉打卡的场景，我们只用记录有签到的信息(1签到0没签到)
使用bitmap

4.基数统计：集合中不重复元素的总数，使用hyperloglog

127.0.0.1:6379> pfadd hp1 1 3 5 7 9
(integer) 1
127.0.0.1:6379> pfadd hp2 2 4 6 8 5
(integer) 1
127.0.0.1:6379> pfcount hp1
(integer) 5
127.0.0.1:6379> pfmerge distResult hp1 hp2
OK
127.0.0.1:6379> pfcount distResult
(integer) 9
127.0.0.1:6379>

1.2 访问指标

uv：用户访问率，一般理解为客户端ip
pv：网页访问率
DAU: 日活跃用户量，登录或者使用了某个产品的用户数(去重复的登录的用户)
MAU：月活跃用户量

二使用hyperloglog统计某宝uv的访问量案例

2.1 使用java代码实现

2.1.1 模拟用户不停访问某宝的数据

代码：

package com.ljf.redis.service.impl;import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.util.Random;
import java.util.concurrent.TimeUnit;/*** @auther zzyy* @create 2022-12-25 11:56*/
@Service
@Slf4j
public class HyperLogLogService
{@Resourceprivate RedisTemplate redisTemplate;/*** 模拟后台有用户点击淘宝首页www.taobao.com，每个用户来自不同的IP地址*/@PostConstructpublic void initIP(){new Thread(() -> {String ip = null;for (int i = 0; i < 20; i++) {Random random = new Random();ip = random.nextInt(256)+"."+random.nextInt(256)+"."+random.nextInt(256)+"."+random.nextInt(256);Long hll = redisTemplate.opsForHyperLogLog().add("hll", ip);log.info("ip={},该IP地址访问首页的次数={}",ip,hll);//暂停3秒钟try { TimeUnit.SECONDS.sleep(3); } catch (InterruptedException e) { e.printStackTrace(); }}},"t1").start();}public long uv(){//PFCOUNTreturn redisTemplate.opsForHyperLogLog().size("hll");}}

2.1.2 查看代码编写

代码

package com.ljf.redis.controller;import com.ljf.redis.service.impl.HyperLogLogService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;import javax.annotation.Resource;/*** @auther zzyy* @create 2022-12-25 11:56*/
@Api(tags = "淘宝亿级UV的Redis统计方案")
@RestController
@Slf4j
public class HyperLogLogController
{@ResourceHyperLogLogService hyperLogLogService;@ApiOperation("获得IP去重复后的UV统计访问量")@RequestMapping(value = "/uv",method = RequestMethod.GET)public long uv(){return hyperLogLogService.uv();}
}

2.2 启动redis服务

1.启动redis服务

[root@localhost ~]# redis-server /myredis/redis.conf
[root@localhost ~]# redis-cli -a 123456 -p 6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> keys *

2.关闭防火墙

2.3 查看

1.页面访问

2.redis查看

三使用GEO获取指定位置附近的坐标

3.1 java实现

1.controller

package com.ljf.redis.controller;import com.ljf.redis.service.impl.GeoService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.geo.Distance;
import org.springframework.data.geo.GeoResults;
import org.springframework.data.geo.Point;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;import javax.annotation.Resource;/*** @auther zzyy* @create 2022-12-25 12:12*/
@Api(tags = "美团地图位置附近的酒店推送GEO")
@RestController
@Slf4j
public class GeoController
{@Resourceprivate GeoService geoService;@ApiOperation("添加坐标geoadd")@RequestMapping(value = "/geoadd",method = RequestMethod.GET)public String geoAdd(){return geoService.geoAdd();}@ApiOperation("获取经纬度坐标geopos")@RequestMapping(value = "/geopos",method = RequestMethod.GET)public Point position(String member){return geoService.position(member);}@ApiOperation("获取经纬度生成的base32编码值geohash")@RequestMapping(value = "/geohash",method = RequestMethod.GET)public String hash(String member){return geoService.hash(member);}@ApiOperation("获取两个给定位置之间的距离")@RequestMapping(value = "/geodist",method = RequestMethod.GET)public Distance distance(String member1, String member2){return geoService.distance(member1,member2);}@ApiOperation("通过经度纬度查找北京王府井附近的")@RequestMapping(value = "/georadius",method = RequestMethod.GET)public GeoResults radiusByxy(){return geoService.radiusByxy();}@ApiOperation("通过地方查找附近,本例写死天安门作为地址,作为家庭作业")@RequestMapping(value = "/georadiusByMember",method = RequestMethod.GET)public GeoResults radiusByMember(){return geoService.radiusByMember();}}

2.servie

package com.ljf.redis.service.impl;import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.geo.*;
import org.springframework.data.redis.connection.RedisGeoCommands;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;import java.util.HashMap;
import java.util.List;
import java.util.Map;/*** @auther zzyy* @create 2022-12-25 12:11*/
@Service
@Slf4j
public class GeoService
{public static final String CITY ="city";@Autowiredprivate RedisTemplate redisTemplate;public String geoAdd(){Map<String, Point> map = new HashMap<>();map.put("天安门",new Point(116.403963,39.915119));map.put("故宫",new Point(116.403414,39.924091));map.put("长城",new Point(116.024067,40.362639));redisTemplate.opsForGeo().add(CITY,map);return map.toString();}public Point position(String member) {//获取经纬度坐标List<Point> list = redisTemplate.opsForGeo().position(CITY, member);System.out.println("获取经纬度....");return list.get(0);}public String hash(String member) {//geohash算法生成的base32编码值List<String> list = redisTemplate.opsForGeo().hash(CITY, member);return list.get(0);}public Distance distance(String member1, String member2) {//获取两个给定位置之间的距离Distance distance = redisTemplate.opsForGeo().distance(CITY, member1, member2,RedisGeoCommands.DistanceUnit.KILOMETERS);return distance;}public GeoResults radiusByxy() {//通过经度，纬度查找附近的,北京王府井位置116.418017,39.914402Circle circle = new Circle(116.418017, 39.914402, Metrics.KILOMETERS.getMultiplier());// 返回50条RedisGeoCommands.GeoRadiusCommandArgs args = RedisGeoCommands.GeoRadiusCommandArgs.newGeoRadiusArgs().includeDistance().includeCoordinates().sortDescending().limit(50);GeoResults<RedisGeoCommands.GeoLocation<String>> geoResults = redisTemplate.opsForGeo().radius(CITY, circle, args);return geoResults;}public GeoResults radiusByMember() {//通过地方查找附近return null;}
}