Springboot 之 Filter 實現 Gzip 壓縮超大 json 物件
2022-10-07 15:00:33
簡介
在專案中，存在傳遞超大 json 資料的場景。直接傳輸超大 json 資料的話，有以下兩個弊端
-
佔用網路頻寬，而有些雲產品就是按照頻寬來計費的，間接浪費了錢
-
傳輸資料大導致網路傳輸耗時較長
為了避免直接傳輸超大 json 資料，可以對 json 資料進行 Gzip 壓縮後，再進行網路傳輸。 -
請求頭新增 Content-Encoding 標識，傳輸的資料進行過壓縮
-
Servlet Filter 攔截請求，對壓縮過的資料進行解壓
-
HttpServletRequestWrapper 包裝，把解壓的資料寫入請求體
pom.xml 引入依賴
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.olive</groupId>
<artifactId>request-uncompression</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>request-uncompression</name>
<url>http://maven.apache.org</url>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.14</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>2.0.14</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.9.0</version>
</dependency>
</dependencies>
</project>
建立壓縮工具類
GzipUtils 類提供壓縮解壓相關方法
package com.olive.utils;
import com.alibaba.fastjson2.JSON;
import com.olive.vo.ArticleRequestVO;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import java.io.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@Slf4j
public class GzipUtils {
private static final String GZIP_ENCODE_UTF_8 = "UTF-8";
/**
* 字串壓縮為GZIP位元組陣列
*
* @param str
* @return
*/
public static byte[] compress(String str) {
return compress(str, GZIP_ENCODE_UTF_8);
}
/**
* 字串壓縮為GZIP位元組陣列
*
* @param str
* @param encoding
* @return
*/
public static byte[] compress(String str, String encoding) {
if (str == null || str.length() == 0) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip = null;
try {
gzip = new GZIPOutputStream(out);
gzip.write(str.getBytes(encoding));
} catch (IOException e) {
log.error("compress>>", e);
}finally {
if(gzip!=null){
try {
gzip.close();
} catch (IOException e) {
}
}
}
return out.toByteArray();
}
/**
* GZIP解壓縮
*
* @param bytes
* @return
*/
public static byte[] uncompress(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
GZIPInputStream unGzip = null;
try {
unGzip = new GZIPInputStream(in);
byte[] buffer = new byte[256];
int n;
while ((n = unGzip.read(buffer)) >= 0) {
out.write(buffer, 0, n);
}
} catch (IOException e) {
log.error("uncompress>>", e);
}finally {
if(unGzip!=null){
try {
unGzip.close();
} catch (IOException e) {
}
}
}
return out.toByteArray();
}
/**
* 解壓並返回String
*
* @param bytes
* @return
*/
public static String uncompressToString(byte[] bytes) throws IOException {
return uncompressToString(bytes, GZIP_ENCODE_UTF_8);
}
/**
* @param bytes
* @return
*/
public static byte[] uncompressToByteArray(byte[] bytes) throws IOException {
return uncompressToByteArray(bytes, GZIP_ENCODE_UTF_8);
}
/**
* 解壓成字串
*
* @param bytes 壓縮後的位元組陣列
* @param encoding 編碼方式
* @return 解壓後的字串
*/
public static String uncompressToString(byte[] bytes, String encoding) throws IOException {
byte[] result = uncompressToByteArray(bytes, encoding);
return new String(result);
}
/**
* 解壓成位元組陣列
*
* @param bytes
* @param encoding
* @return
*/
public static byte[] uncompressToByteArray(byte[] bytes, String encoding) throws IOException {
if (bytes == null || bytes.length == 0) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
GZIPInputStream unGzip = null;
try {
unGzip = new GZIPInputStream(in);
byte[] buffer = new byte[256];
int n;
while ((n = unGzip.read(buffer)) >= 0) {
out.write(buffer, 0, n);
}
return out.toByteArray();
} catch (IOException e) {
log.error("uncompressToByteArray>>", e);
throw new IOException("解壓縮失敗！");
}finally {
if(unGzip!=null){
unGzip.close();
}
}
}
/**
* 將位元組流轉換成檔案
*
* @param filename
* @param data
* @throws Exception
*/
public static void saveFile(String filename, byte[] data) throws Exception {
FileOutputStream fos = null;
try {
if (data != null) {
String filepath = "/" + filename;
File file = new File(filepath);
if (file.exists()) {
file.delete();
}
fos = new FileOutputStream(file);
fos.write(data, 0, data.length);
fos.flush();
System.out.println(file);
}
}catch (Exception e){
throw e;
}finally {
if(fos!=null){
fos.close();
}
}
}
}
對Request進行包裝
UnZipRequestWrapper 讀取輸入流，然進行解壓；解壓完後，再把解壓出來的資料封裝到輸入流中。
package com.olive.filter;
import com.olive.utils.GzipUtils;
import lombok.extern.slf4j.Slf4j;
import javax.servlet.ReadListener;
import javax.servlet.ServletInputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import java.io.*;
/**
* Json String 經過壓縮後儲存為二進位制檔案 -> 解壓縮後還原成 Jso nString轉換成byte[]寫回body中
*/
@Slf4j
public class UnZipRequestWrapper extends HttpServletRequestWrapper {
private final byte[] bytes;
public UnZipRequestWrapper(HttpServletRequest request) throws IOException {
super(request);
try (BufferedInputStream bis = new BufferedInputStream(request.getInputStream());
ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
final byte[] body;
byte[] buffer = new byte[1024];
int len;
while ((len = bis.read(buffer)) > 0) {
baos.write(buffer, 0, len);
}
body = baos.toByteArray();
if (body.length == 0) {
log.info("Body無內容，無需解壓");
bytes = body;
return;
}
this.bytes = GzipUtils.uncompressToByteArray(body);
} catch (IOException ex) {
log.error("解壓縮步驟發生異常！", ex);
throw ex;
}
}
@Override
public ServletInputStream getInputStream() throws IOException {
final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
return new ServletInputStream() {
@Override
public boolean isFinished() {
return false;
}
@Override
public boolean isReady() {
return false;
}
@Override
public void setReadListener(ReadListener readListener) {
}
public int read() throws IOException {
return byteArrayInputStream.read();
}
};
}
@Override
public BufferedReader getReader() throws IOException {
return new BufferedReader(new InputStreamReader(this.getInputStream()));
}
}
定義GzipFilter對請求進行攔截
GzipFilter 攔截器根據請求頭是否包含Content-Encoding=application/gzip，如果包含就對資料進行解壓；否則就直接放過。
package com.olive.filter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
/**
* 解壓filter
*/
@Slf4j
@Component
public class GzipFilter implements Filter {
private static final String CONTENT_ENCODING = "Content-Encoding";
private static final String CONTENT_ENCODING_TYPE = "application/gzip";
@Override
public void init(FilterConfig filterConfig) throws ServletException {
log.info("init GzipFilter");
}
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
long start = System.currentTimeMillis();
HttpServletRequest httpServletRequest = (HttpServletRequest)request;
String encodeType = httpServletRequest.getHeader(CONTENT_ENCODING);
if (encodeType!=null && CONTENT_ENCODING_TYPE.equals(encodeType)) {
log.info("請求:{} 需要解壓", httpServletRequest.getRequestURI());
UnZipRequestWrapper unZipRequest = new UnZipRequestWrapper(httpServletRequest);
chain.doFilter(unZipRequest, response);
}else {
log.info("請求:{} 無需解壓", httpServletRequest.getRequestURI());
chain.doFilter(request,response);
}
log.info("耗時：{}ms", System.currentTimeMillis() - start);
}
@Override
public void destroy() {
log.info("destroy GzipFilter");
}
}
註冊 GzipFilter 攔截器
package com.olive.config;
import com.olive.filter.GzipFilter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* 註冊filter
*/
@Configuration
public class FilterRegistration {
@Autowired
private GzipFilter gzipFilter;
@Bean
public FilterRegistrationBean<GzipFilter> gzipFilterRegistrationBean() {
FilterRegistrationBean<GzipFilter> registration = new FilterRegistrationBean<>();
//Filter可以new，也可以使用依賴注入Bean
registration.setFilter(gzipFilter);
//過濾器名稱
registration.setName("gzipFilter");
//攔截路徑
registration.addUrlPatterns("/*");
//設定順序
registration.setOrder(1);
return registration;
}
}
定義 Controller
該 Controller 非常簡單，主要是輸入請求的資料
package com.olive.controller;
import java.util.HashMap;
import java.util.Map;
import com.alibaba.fastjson2.JSON;
import com.olive.vo.ArticleRequestVO;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class TestController {
@RequestMapping("/getArticle")
public Map<String, Object> getArticle(@RequestBody ArticleRequestVO articleRequestVO){
Map<String, Object> result = new HashMap<>();
result.put("code", 200);
result.put("msg", "success");
System.out.println(JSON.toJSONString(articleRequestVO));
return result;
}
}
Controller 引數接收VO
package com.olive.vo;
import lombok.Data;
import java.io.Serializable;
@Data
public class ArticleRequestVO implements Serializable {
private Long id;
private String title;
private String content;
}
定義 Springboot 引導類
package com.olive;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class);
}
}
測試
- 非壓縮請求測試
curl -X POST \
http://127.0.0.1:8080/getArticle \
-H 'content-type: application/json' \
-d '{
"id":1,
"title": "java樂園",
"content":"xxxxxxxxxx"
}'
- 壓縮請求測試
不要直接將壓縮後的 byte[] 陣列當作字串進行傳輸，否則壓縮後的請求資料比沒壓縮後的還要大得多！
專案中一般採用以下兩種傳輸壓縮後的 byte[] 的方式：
- 將壓縮後的 byet[] 進行 Base64 編碼再傳輸字串，這種方式會損失掉一部分 GZIP 的壓縮效果，適用於壓縮結果要儲存在 Redis 中的情況
- 將壓縮後的 byte[] 以二進位制的形式寫入到檔案中，請求時直接在 body 中帶上檔案即可，用這種方式可以不損失壓縮效果
小編測試採用第二種方式，採用以下程式碼把原始資料進行壓縮
public static void main(String[] args) {
ArticleRequestVO vo = new ArticleRequestVO();
vo.setId(1L);
vo.setTitle("bug弄潮兒");
try {
byte[] bytes = FileUtils.readFileToByteArray(new File("C:\\Users\\2230\\Desktop\\凱平專案資料\\改裝車專案\\CXSSBOOT_DB_DDL-1.0.9.sql"));
vo.setContent(new String(bytes));
byte[] dataBytes = compress(JSON.toJSONString(vo));
saveFile("d:/vo.txt", dataBytes);
} catch (Exception e) {
e.printStackTrace();
}
}
壓縮後資料儲存到d:/vo.txt，然後在 postman 中安裝下圖選擇
