当前位置： 首页 > java >正文

常见压缩算法性能和压缩率对比 LZ4 LZO ZSTD SNAPPY

java 2025/7/19 1:03:59

网传压缩算法对比表

算法压缩率压缩速度解压速度支持流式压缩适用场景
LZ4低极快极快是实时数据压缩、日志压缩、内存缓存等
Zstandard高快快是文件压缩、网络传输、数据库备份等
Brotli很高中等快是静态资源压缩（HTML、CSS、JS）等
LZO低极快快是嵌入式系统、实时数据压缩
Snappy中等快快否日志压缩、内存缓存、网络传输

性能实测

在这里插入图片描述

./compress_test syslog.txtLZ4 0 comp:   64.669 ms  decom:   15.282 ms  Size: 85.906 MB  Ratio: 17.186 %
LZ4 1 comp:   65.662 ms  decom:   15.046 ms  Size: 85.906 MB  Ratio: 17.186 %
LZO 0 comp:   65.834 ms  decom:   51.929 ms  Size: 85.906 MB  Ratio: 17.936 %
LZO 1 comp:   61.834 ms  decom:   50.462 ms  Size: 85.906 MB  Ratio: 17.936 %
Zstd 0 comp:   71.713 ms  decom:   31.642 ms  Size: 85.906 MB  Ratio: 9.230 %
Zstd 1 comp:   93.255 ms  decom:   47.457 ms  Size: 85.906 MB  Ratio: 9.230 %
Brotli 0 comp:  156.201 ms  decom:  115.879 ms  Size: 85.906 MB  Ratio: 13.189 %
Brotli 1 comp:  127.775 ms  decom:  104.258 ms  Size: 85.906 MB  Ratio: 13.189 %
Snappy 0 comp:   84.727 ms  decom:   52.499 ms  Size: 85.906 MB  Ratio: 18.220 %
Snappy 1 comp:   82.503 ms  decom:   52.898 ms  Size: 85.906 MB  Ratio: 18.220 %

对于一个系统日志文件，LZ4 LZO速度最快， 压缩率zstd遥遥领先，综合性价比 Zstd完胜。

安装与使用

LZ4

编译

git clone https://github.com/lz4/lz4.git
cd lz4
make

使用

ckun@DESKTOP-PODAH1A:~/ws/compress/test$ rm -f log.lz4
ckun@DESKTOP-PODAH1A:~/ws/compress/test$ time ./lz4 syslog -T1 log.lz4
Compressed 3290808339 bytes into 567183342 bytes ==> 17.24%real    0m5.855s
user    0m4.636s
sys     0m1.910s

Zstd

git clone https://github.com/facebook/zstd.git
cd zstd
make

Brotli

git clone https://github.com/google/brotli.git
cd brotlimkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make

LZO

wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
tar -xvzf lzo-2.10.tar.gz
cd lzo-2.10
mkdir build
cd build
cmake -DENABLE_SHARED=ON ..
make

Snappy

git clone https://github.com/google/snappy.git
cd snappy
git submodule update --init
mkdir build
cd build 
cmake ..
make

测试程序


#include <iostream>
#include <fstream>
#include <vector>
#include <chrono>
#include <cstring>
#include <lz4.h>
#include <lzo/lzo1x.h>
#include <zstd.h>
#include <brotli/encode.h>
#include <brotli/decode.h>
#include <snappy.h>#include <iomanip> 
// Helper function to calculate checksum (simple Adler-32 implementation)
uint32_t calculate_checksum(const std::vector<char>& data) {uint32_t a = 1, b = 0;for (char c : data) {a = (a + static_cast<uint8_t>(c)) % 65521;b = (b + a) % 65521;}return (b << 16) | a;
}// Read file into memory
std::vector<char> read_file(const std::string& filename) {std::ifstream file(filename, std::ios::binary | std::ios::ate);if (!file) {throw std::runtime_error("Failed to open file: " + filename);}std::streamsize size = file.tellg();file.seekg(0, std::ios::beg);std::vector<char> buffer(size);if (!file.read(buffer.data(), size)) {throw std::runtime_error("Failed to read file: " + filename);}return buffer;
}// Test LZ4 compression and decompression
void test_lz4(const std::vector<char>& input, int i) {//std::cout << "Testing LZ4..." << std::endl;// Allocate buffersint max_compressed_size = LZ4_compressBound(input.size());std::vector<char> compressed(max_compressed_size);std::vector<char> decompressed(input.size());// Compressauto start = std::chrono::high_resolution_clock::now();int compressed_size = LZ4_compress_default(input.data(), compressed.data(), input.size(), max_compressed_size);auto end = std::chrono::high_resolution_clock::now();if (compressed_size <= 0) {throw std::runtime_error("LZ4 compression failed");}auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();int decompressed_size = LZ4_decompress_safe(compressed.data(), decompressed.data(), compressed_size, decompressed.size());end = std::chrono::high_resolution_clock::now();if (decompressed_size != input.size()) {throw std::runtime_error("LZ4 decompression failed");}auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(decompressed);if (original_checksum != decompressed_checksum) {throw std::runtime_error("LZ4 checksum mismatch");}// Output resultsprintf("LZ4 %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}// Test LZO compression and decompression
void test_lzo(const std::vector<char>& input, int i) {//std::cout << "Testing LZO..." << std::endl;// Initialize LZOif (lzo_init() != LZO_E_OK) {throw std::runtime_error("LZO initialization failed");}// Allocate buffersstd::vector<char> compressed(input.size() + input.size() / 16 + 64 + 3);std::vector<char> decompressed(input.size());std::vector<unsigned char> wrkmem(LZO1X_1_MEM_COMPRESS);// Compressauto start = std::chrono::high_resolution_clock::now();lzo_uint compressed_size;int r = lzo1x_1_compress(reinterpret_cast<const unsigned char*>(input.data()), input.size(),reinterpret_cast<unsigned char*>(compressed.data()), &compressed_size, wrkmem.data());auto end = std::chrono::high_resolution_clock::now();if (r != LZO_E_OK) {throw std::runtime_error("LZO compression failed");}auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();lzo_uint decompressed_size;r = lzo1x_decompress(reinterpret_cast<const unsigned char*>(compressed.data()), compressed_size,reinterpret_cast<unsigned char*>(decompressed.data()), &decompressed_size, nullptr);end = std::chrono::high_resolution_clock::now();if (r != LZO_E_OK || decompressed_size != input.size()) {throw std::runtime_error("LZO decompression failed");}auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(decompressed);if (original_checksum != decompressed_checksum) {throw std::runtime_error("LZO checksum mismatch");}// Output results// std::cout << "LZO " << i << " ";// std::cout << "comp: " << (compression_time / 1000.0) << " ms  "//           << "decom: " << (decompression_time / 1000.0) << " ms  "//           << "Size: " << (input.size() / (1024.0 * 1024.0)) << " MB  "//           << "Ratio: " << (static_cast<double>(compressed_size) / input.size()) * 100 << " %" //           << std::endl;printf("LZO %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}void test_zstd(const std::vector<char>& input, int i) {// Allocate bufferssize_t max_compressed_size = ZSTD_compressBound(input.size());std::vector<char> compressed(max_compressed_size);std::vector<char> decompressed(input.size());// Compressauto start = std::chrono::high_resolution_clock::now();size_t compressed_size = ZSTD_compress(compressed.data(), max_compressed_size, input.data(), input.size(), 1);auto end = std::chrono::high_resolution_clock::now();if (ZSTD_isError(compressed_size)) {throw std::runtime_error("Zstd compression failed: " + std::string(ZSTD_getErrorName(compressed_size)));}auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();size_t decompressed_size = ZSTD_decompress(decompressed.data(), decompressed.size(), compressed.data(), compressed_size);end = std::chrono::high_resolution_clock::now();if (ZSTD_isError(decompressed_size) || decompressed_size != input.size()) {throw std::runtime_error("Zstd decompression failed: " + std::string(ZSTD_getErrorName(decompressed_size)));}auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(decompressed);if (original_checksum != decompressed_checksum) {throw std::runtime_error("Zstd checksum mismatch");}// Output resultsprintf("Zstd %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}void test_brotli(const std::vector<char>& input, int i) {// Allocate bufferssize_t max_compressed_size = BrotliEncoderMaxCompressedSize(input.size());std::vector<uint8_t> compressed(max_compressed_size);std::vector<uint8_t> decompressed(input.size());// Compressauto start = std::chrono::high_resolution_clock::now();size_t compressed_size = max_compressed_size;if (!BrotliEncoderCompress(0, BROTLI_DEFAULT_WINDOW, BROTLI_MODE_GENERIC,input.size(), reinterpret_cast<const uint8_t*>(input.data()),&compressed_size, compressed.data())) {throw std::runtime_error("Brotli compression failed");}auto end = std::chrono::high_resolution_clock::now();auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();size_t decompressed_size = decompressed.size();if (BrotliDecoderDecompress(compressed_size, compressed.data(),&decompressed_size, decompressed.data()) != BROTLI_DECODER_RESULT_SUCCESS) {throw std::runtime_error("Brotli decompression failed");}end = std::chrono::high_resolution_clock::now();auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(std::vector<char>(decompressed.begin(), decompressed.end()));if (original_checksum != decompressed_checksum) {throw std::runtime_error("Brotli checksum mismatch");}// Output resultsprintf("Brotli %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}void test_snappy(const std::vector<char>& input, int i) {// Allocate buffersstd::string compressed;std::string decompressed;// Compressauto start = std::chrono::high_resolution_clock::now();snappy::Compress(input.data(), input.size(), &compressed);auto end = std::chrono::high_resolution_clock::now();auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();if (!snappy::Uncompress(compressed.data(), compressed.size(), &decompressed)) {throw std::runtime_error("Snappy decompression failed");}end = std::chrono::high_resolution_clock::now();auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(std::vector<char>(decompressed.begin(), decompressed.end()));if (original_checksum != decompressed_checksum) {throw std::runtime_error("Snappy checksum mismatch");}// Output resultsprintf("Snappy %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed.size()) / input.size()) * 100);
}template <typename Func>
void run_test(Func func, const std::vector<char>& input, int times) {for (int i = 0; i < times; ++i) {func(input, i);}
}int main(int argc, char* argv[]) {if (argc != 2) {std::cerr << "Usage: " << argv[0] << " <file>" << std::endl;return 1;}std::cout << std::fixed << std::setprecision(3); try {// Read file into memorystd::vector<char> input = read_file(argv[1]);// Calculate checksumuint32_t checksum = calculate_checksum(input);std::cout << "Original Checksum: " << checksum << std::endl;// Test LZ4run_test(test_lz4, input, 2);// Test LZOrun_test(test_lzo, input, 2);run_test(test_zstd, input, 2);// Test Brotlirun_test(test_brotli, input, 2);// Test Snappyrun_test(test_snappy, input, 2);} catch (const std::exception& e) {std::cerr << "Error: " << e.what() << std::endl;return 1;}return 0;
}

Makefile

TARGET = compress_testSRCS = $(wildcard *.cpp)CC = g++CFLAGS += -g -O2 -I../lz4/lib -I../lzo-2.10/include -I../zstd/lib \-I../brotli/c/include -I../snappy LDFLAGS += -lyaml-cpp  -L../lz4/lib -llz4 \-L../lzo-2.10/build -llzo2 -lz \-L../zstd/lib -lzstd \-L../brotli/build -lbrotlicommon -lbrotlidec -lbrotlienc \-L../snappy/build -lsnappy
all: $(TARGET)$(TARGET): $(SRCS)$(CC) $(CFLAGS) $(SRCS) -o $@ $(LDFLAGS).PHONY: clean
clean:rm $(TARGET)

运行

export LD_LIBRARY_PATH=../lz4/lib:../lzo-2.10/build:../zstd/lib:../brotli/build/:../snappy/build$ ldd compress_testlinux-vdso.so.1 (0x00007ffdb71cf000)liblz4.so.1 => ../lz4/lib/liblz4.so.1 (0x00007fc26045a000)liblzo2.so.2 => ../lzo-2.10/build/liblzo2.so.2 (0x00007fc260427000)libzstd.so.1 => ../zstd/lib/libzstd.so.1 (0x00007fc260330000)libbrotlidec.so.1 => ../brotli/build/libbrotlidec.so.1 (0x00007fc260321000)libbrotlienc.so.1 => ../brotli/build/libbrotlienc.so.1 (0x00007fc260268000)libsnappy.so.1 => ../snappy/build/libsnappy.so.1 (0x00007fc260259000)libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007fc260026000)libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007fc260006000)libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fc25fddd000)libbrotlicommon.so.1 => ../brotli/build/libbrotlicommon.so.1 (0x00007fc25fdb7000)libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fc25fcd0000)/lib64/ld-linux-x86-64.so.2 (0x00007fc260498000)./compress_test file
http://www.xdnf.cn/news/10052.html

相关文章：

  • Haproxy搭建web群集
  • WWW22-可解释推荐|用于推荐的神经符号描述性规则学习
  • 【免费的高清录屏软件】OBS Studio
  • 架构加速-深度学习教程
  • A类地址中最小网络号（0.x.x.x） 默认路由 / 无效/未指定地址
  • Qt中使用正则表达式来提取字符串
  • 第100+41步 ChatGPT学习：R语言实现误判病例分析
  • Windows 权限提升 | TryHackMe | Windows Privilege Escalation
  • html中使用nginx ssi插入html
  • 全志科技携飞凌嵌入式T527核心板亮相OpenHarmony开发者大会
  • 智能守护电网安全：探秘输电线路测温装置的科技力量
  • MySQL高可用集群
  • 《Linux C编程实战》笔记：套接字编程
  • day41 python图像识别任务
  • 【多线程初阶】线程状态 线程安全
  • 进阶智能体实战九、图文需求分析助手（ChatGpt多模态版）（帮你生成 模块划分+页面+表设计、状态机、工作流、ER模型）
  • Ubuntu 安装 FSL 及多模态脑MRI的去颅骨处理（含 HD-BET 深度学习方法）
  • 区域未停留检测算法AI智能分析网关V4打造铁道/工厂/机场等场景应用方案
  • mysql隐式转换会造成索引失效的原因
  • 软件评测机构如何保障质量？检测资质、技术实力缺一不可
  • 历年浙江大学计算机保研上机真题
  • JavaScript 性能优化实战研讨
  • antDesignVue中a-upload上传组件的使用
  • Ubuntu开机自动运行Docker容器中的Qt UI程序
  • redis持久化策略
  • ansible自动化playbook简单实践
  • 从监控到告警：Prometheus+Grafana+Alertmanager+告警通知服务全链路落地实践
  • 湖北理元理律师事务所：债务优化中的生活保障实践
  • Java—— 多线程 第二期
  • 新松机械臂 2001端口服务的客户端例程