gateway中对返回的数据进行处理
- 背景
- 1.项目层次
背景
最近公司有个需求是对返回数据进行处理，比如进行数据脱敏。最后在gateway中进行处理。
1.项目层次
根据项目的结构，原本在菜单功能处有对于权限设计的url判断，所以在url后面加了一个正则表达式的字段，例如“/^(1[3-9][0-9])\d{4}(\d{4}$)/:$1****$2”
因为正则表达式，我存储在redis和本地缓存中，表达式中的转义符号一定要注意，我在处理时，处理了转义，所以在页面填写时需要多加
```java
package com.qlisv.qqdznyyglpt.gateway.filter;
import cn.hutool.json.JSONUtil;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.qlisv.qqdznyyglpt.gateway.feign.PermissionsClient;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.reactivestreams.Publisher;
import org.springframework.cloud.gateway.filter.GatewayFilterChain;
import org.springframework.cloud.gateway.filter.GlobalFilter;
import org.springframework.core.Ordered;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferFactory;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.server.reactive.ServerHttpRequest;
import org.springframework.http.server.reactive.ServerHttpResponse;
import org.springframework.http.server.reactive.ServerHttpResponseDecorator;
import org.springframework.stereotype.Component;
import org.springframework.util.AntPathMatcher;
import org.springframework.util.CollectionUtils;
import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import javax.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
/**
* @author 63418
*/
@Slf4j
@Component
public class DataDesensitizationResponseFilter implements GlobalFilter, Ordered {
@Resource
private PermissionsClient permissionsClient;
@Resource
private RedisTemplate redisTemplate;
public static final String redisDataDesensitizationKey = "redisDataDesensitizationKey:%s:%s";
public static Cache<String, Object> cache = CacheBuilder.newBuilder()
// 初始容量
.initialCapacity(5)
// 最大缓存数，超出淘汰
.maximumSize(50)
// 过期时间 设置写入3秒后过期
.expireAfterWrite(60, TimeUnit.SECONDS)
.build();
@Override
public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
log.info("DataDesensitizationResponseFilter---path={}", exchange.getRequest().getPath());
if (exchange.getRequest().getMethod() == HttpMethod.OPTIONS) {
return chain.filter(exchange);
}
boolean isExternalRequest = isExternalRequest(exchange);
// boolean checkContentTypeAndUrl = checkContentTypeAndUrl(exchange);
if (isExternalRequest) {
// 如果是外部请求，处理返回的数据
return processResponse(exchange, chain);
}
return chain.filter(exchange);
}
private boolean isExternalRequest(ServerWebExchange exchange) {
// 判断请求是否来自外部
// 可以根据实际情况进行自定义判断.这个判断基于nginx代理，如果后面有外部程序调用，根据场景修改此处判断,,nginx的配置文件一定要加上X-Nginx-Proxy这个header
ServerHttpRequest request = exchange.getRequest();
if (request.getHeaders().containsKey("X-Nginx-Proxy") &&
request.getHeaders().containsKey("X-Real-IP")) {
return true;
}
return false;
}
private Mono<Void> processResponse(ServerWebExchange exchange, GatewayFilterChain chain) {
ServerHttpResponse originalResponse = exchange.getResponse();
DataBufferFactory bufferFactory = originalResponse.bufferFactory();
try {
ServerHttpResponseDecorator decoratedResponse = new ServerHttpResponseDecorator(originalResponse) {
@Override
public Mono<Void> writeWith(Publisher<? extends DataBuffer> body) {
MediaType contentType = this.getHeaders().getContentType();
String contentTypeString = contentType.toString();
if (!contentTypeString.startsWith(MediaType.APPLICATION_JSON_VALUE) || !exchange.getResponse().getStatusCode().equals(HttpStatus.OK)) {
return super.writeWith(body);
}
// 异步获取会话中的属性
return exchange.getSession()
.flatMap(session -> {
// 获取会话中的 username 和 tenantId
String username = session.getAttribute("username");
String tenantId = session.getAttribute("tenantId");
String userId = session.getAttribute("userId");
// 创建一个Map，包含 username 和 tenantId
Map<String, String> name = new HashMap<>();
name.put("username", username);
name.put("tenantId", tenantId);
name.put("userId", userId);
// 返回这个Map
return Mono.just(name);
}).flatMap(nameMap -> {
Map<String, String> dataDesensitizationMap = new LinkedHashMap<>();
try {
dataDesensitizationMap = getDataDesensitizationMap(nameMap.get("username"), nameMap.get("userId"), nameMap.get("tenantId"));
if (!checkDataDesensitizationMap(dataDesensitizationMap, exchange)) {
return super.writeWith(body);
}
} catch (Exception e) {
log.error("[DataDesensitizationResponseFilter] 获取配置和正则表达式异常", e);
return super.writeWith(body);
}
if (body instanceof Flux) {
Flux<? extends DataBuffer> fluxBody = (Flux<? extends DataBuffer>) body;
Map<String, String> finalDataDesensitizationMap = dataDesensitizationMap;
return super.writeWith(fluxBody.buffer().map(dataBuffers -> {
byte[] newContent = new byte[0];
try {
DataBufferFactory dataBufferFactory = new DefaultDataBufferFactory();
DataBuffer join = dataBufferFactory.join(dataBuffers);
byte[] content = new byte[join.readableByteCount()];
join.read(content);
DataBufferUtils.release(join);
// 获取响应数据
String responseStr = new String(content, StandardCharsets.UTF_8);
// 获取响应数据
List<String> strings = exchange.getResponse().getHeaders().get(HttpHeaders.CONTENT_ENCODING);
if (!CollectionUtils.isEmpty(strings) && strings.contains("gzip")) {
GZIPInputStream gzipInputStream = null;
try {
gzipInputStream = new GZIPInputStream(new ByteArrayInputStream(content), content.length);
StringWriter writer = new StringWriter();
IOUtils.copy(gzipInputStream, writer, StandardCharsets.UTF_8);
responseStr = writer.toString();
} catch (IOException e) {
log.error("====Gzip IO error", e);
} finally {
if (gzipInputStream != null) {
try {
gzipInputStream.close();
} catch (IOException e) {
log.error("===Gzip IO close error", e);
}
}
}
} else {
responseStr = new String(content, StandardCharsets.UTF_8);
}
newContent = desensitizeJson(responseStr, finalDataDesensitizationMap).getBytes(StandardCharsets.UTF_8);
originalResponse.getHeaders().setContentLength(newContent.length);
} catch (Exception e) {
log.error("responseStr exchange error", e);
throw new RuntimeException(e);
}
return bufferFactory.wrap(newContent);
}));
}
return super.writeWith(body);
});
}
@Override
public Mono<Void> writeAndFlushWith(Publisher<? extends Publisher<? extends DataBuffer>> body) {
return writeWith(Flux.from(body).flatMapSequential(p -> p));
}
};
return chain.filter(exchange.mutate().response(decoratedResponse).build());
} catch (Exception e) {
log.error("RewriteResponse error", e);
return chain.filter(exchange);
}
}
private static String desensitizeJson(String json, Map<String, String> dataDesensitization) {
// 在这里实现你的 JSON 脱敏逻辑
// 例如，使用 JSON 库解析 JSON，修改需要脱敏的字段，然后再序列化回 JSON 字符串
// 返回脱敏后的 JSON 字符串
String regularExpression = dataDesensitization.get("RegularExpression");
String[] regularExpressionArr = regularExpression.split(";");
for (String regularExpressionStr : regularExpressionArr) {
String[] regularExpressionStrArr = regularExpressionStr.split(":");
String regularExpressionStrKey = regularExpressionStrArr[0];
String regularExpressionStrValue = regularExpressionStrArr[1];
//这个地方的转义字符好坑，，，，
Pattern pattern = Pattern.compile(StringEscapeUtils.unescapeJava(regularExpressionStrKey));
Matcher matcher = pattern.matcher(json);
json = matcher.replaceAll(StringEscapeUtils.unescapeJava(regularExpressionStrValue));
}
return json;
}
/**
* 通过本地缓存或者redis缓存获取脱敏规则，规则的刷新在CustomServerAuthenticationSuccessHandler ，登录设置session时刷新
* @param username
* @param userId
* @param tenantId
* @return
*/
private Map<String, String> getDataDesensitizationMap(String username, String userId, String tenantId) {
String key = String.format(redisDataDesensitizationKey, tenantId, userId);
Map reDataDesensitization = null;
try {
reDataDesensitization = (Map) cache.get(key, () -> {
if (redisTemplate.hasKey(key)) {
Object dataDesensitization = redisTemplate.opsForValue().get(key);
redisTemplate.expire(key, 60, TimeUnit.MINUTES);
return dataDesensitization;
}
return null;
});
cache.put(key, reDataDesensitization);
if (reDataDesensitization != null) {
return reDataDesensitization;
}
} catch (Exception e) {
log.error("[DataDesensitizationResponseFilter] 从缓存获取数据异常", e);
}
Map<String, String> dataDesensitization = permissionsClient.findDataDesensitizationByUsername(username, userId, tenantId);
cache.put(key, dataDesensitization);
redisTemplate.opsForValue().set(key, dataDesensitization, 60, TimeUnit.MINUTES);
return dataDesensitization;
}
private Boolean checkDataDesensitizationMap(Map<String, String> dataDesensitization, ServerWebExchange exchange) {
if (dataDesensitization == null) {
return false;
}
if (!dataDesensitization.containsKey("data_desensitization")) {
return false;
}
String data_desensitization = dataDesensitization.get("data_desensitization");
if ("false".equals(data_desensitization)) {
return false;
}
ServerHttpRequest request = exchange.getRequest();
// 获取请求的路径
String path = request.getPath().toString();
for (String key : dataDesensitization.keySet()) {
// 如果dataDesensitization Map中的某个键与请求的路径匹配，则返回true
// spring的路径匹配工具，匹配一些特殊写法
AntPathMatcher matcher = new AntPathMatcher();
if (matcher.match(key, path)) {
// 检查对应路径的值是否为true，如果是则表示需要进行数据脱敏
dataDesensitization.put("RegularExpression", dataDesensitization.get(key));
return true;
}
}
return false;
}
@Override
public int getOrder() {
return -2;
}
public static void main(String[] args) {
String str="{\"customerContacts\":[],\"businessData\":[{\"name\":\"测试\",\"id\":\"ab7025db-e61e-4c4f-8dc9-8bf5539e05ad\",\"value\":[[\"ID\",\"姓名\",\"性别\",\"出生日期\",\"备注\",\"权限\"]]},{\"name\":\"预约\",\"id\":\"f3a99a5d-ec79-4523-a632-2dbc6f6bf83f\",\"value\":[[\"ID\",\"社保\",\"公积金\",\"数据\",\"其他\"],[\"87bccd32-7b48-4cb8-88f2-8eaaf7d15e65\",\"二档\",\"是\",\"2024-03-22 11:40:12\",\"李十四\"]]}],\"customerInfoExt\":{\"id\":null,\"tenantId\":null,\"customerId\":null,\"delStatus\":0,\"field1\":null,\"field2\":null,\"field3\":null,\"field4\":null,\"field5\":null,\"field6\":null,\"field7\":null,\"field8\":null,\"field9\":null,\"field10\":null,\"field11\":null,\"field12\":null,\"field13\":null,\"field14\":null,\"field15\":null,\"field16\":null,\"field17\":null,\"field18\":null,\"field19\":null,\"field20\":null,\"field21\":null,\"field22\":null,\"field23\":null,\"field24\":null,\"field25\":null,\"field26\":null,\"field27\":null,\"field28\":null,\"field29\":null,\"field30\":null,\"field31\":null,\"field32\":null,\"field33\":null,\"field34\":null,\"field35\":null,\"field36\":null,\"field37\":null,\"field38\":null,\"field39\":null,\"field40\":null,\"field41\":null,\"field42\":null,\"field43\":null,\"field44\":null,\"field45\":null,\"field46\":null,\"field47\":null,\"field48\":null,\"field49\":null,\"field50\":null},\"customer\":{\"id\":\"972d7770-0d1a-4c5b-a171-5ab4ff144764\",\"tenantId\":\"8654e94c-f430-4574-ab08-6b4bbbad1e9d\",\"customerName\":\"罗棉\",\"customerCode\":\"1\",\"sex\":0,\"age\":24,\"education\":4,\"birthday\":\"2024-03-22\",\"certificateType\":\"0\",\"certificateNo\":\"511***********1023\",\"maritalStatus\":1,\"accountLocation\":\"广东深圳龙华区民治大道2\",\"accountLocationPostal\":\"632200\",\"currentPostal\":\"456789\",\"address\":\"深圳市龙华区民治街道水围小区\",\"agentId\":\"1002\",\"accountManager\":\"李三\",\"email\":\"3570178990@qq.com\",\"qq\":\"12568900\",\"wechat\":\"ql8563515\",\"weiboNumber\":\"12345123\",\"personalTelephone\":\"17890804789\",\"personalMobilePhone\":\"17890890000\",\"officePhone\":\"13956789012\",\"homePhone\":\"12345678901\",\"emergencyContactPhone\":\"15908765439\",\"otherNumber1\":\"1234567890\",\"otherNumber2\":\"一档\",\"otherNumber3\":null,\"otherNumber4\":null,\"otherNumber5\":null,\"industry\":\"1\",\"industryType\":\"3\",\"companyNature\":\"1\",\"companyName\":\"飞牛公司\",\"companyPost\":\"委员\",\"companySize\":\"1\",\"companyAddress\":\"深圳市龙华区民治街道水围小区\",\"companyPostal\":\"636600\",\"companyPhone\":\"15267890562\",\"workingYears\":1.0,\"annualIncome\":null,\"socialSecurity\":1,\"delStatus\":0,\"creater\":\"jmh\",\"createTime\":\"2024-03-22 11:35:57\",\"updater\":\"jmh\",\"updateTime\":\"2024-03-25 17:15:04\",\"customerType\":1,\"customerBusinessId\":\"2010\",\"sourceFrom\":\"1\"}}";
System.out.println(str.replaceAll("/^(1[3-9][0-9])\\d{4}(\\d{4}$)/","$1****$2"));
}
}
