最近在做一个项目，需要将音频送入 AI 模型进行处理。整个流程包括：
从
.wav文件中加载音频；进行 短时傅里叶变换（STFT）；
将变换结果输入模型；
使用 逆STFT（ISTFT） 重建回时域信号。
作为一个长期从事图像方向的 CVer，我对音频领域相对陌生。在调研后发现，和计算机视觉相比，优质的音频处理库屈指可数。而手撸 STFT / ISTFT 又繁琐、容易出错。
最终，我选择了开源音频分析库 Essentia，其功能强大、API 结构清晰，非常适合科研和快速原型验证。本文将分享如何在 C++ 中基于 Essentia 实现 STFT / ISTFT，并完成音频的重建。
🔍 什么是 Essentia？
Essentia 是由巴塞罗那庞培法布拉大学音乐技术小组（MTG）开发的开源 C++ 音频分析库，发布于 AGPLv3 许可下。
Essentia 提供了一套丰富的算法模块，覆盖以下能力：
音频 I/O 和预处理；
数字信号处理（DSP）基础块；
声学和音乐特征提取（如光谱、音调、节奏、情绪等）；
Python 包装与 Vamp 插件支持，便于快速原型与可视化。
虽然资料较少、使用门槛稍高，但其结构良好，非常适合做科研实验或工业应用中的音频前处理部分。
🛠 编译 Essentia（推荐 Docker 环境）
手动编译步骤如下：
git clone https://github.com/MTG/essentia.git
cd essentia
packaging/build_3rdparty_static_debian.sh⚠️ 如果第三方依赖下载失败，可以手动修改 packaging/debian_3rdparty/ 下的对应脚本，将源码下载到指定目录。
接着执行：
./waf configure --with-static-examples
./waf
如果你希望跳过编译，可以从我上传的打包资源中下载使用。
📦 在 C++ 中使用 STFT
AlgorithmFactory& factory = standard::AlgorithmFactory::instance();
// 1. 加载音频
Algorithm* loader = factory.create("MonoLoader",
"filename", "test.wav",
"sampleRate", 16000);
vector<Real> audio;
loader->output("audio").set(audio);
loader->compute();
delete loader;
设置参数：
const int frameSize = 320;
const int hopSize = 80; // 25% overlap
STFT 流程：
Algorithm* frameCutter = factory.create("FrameCutter",
"frameSize", frameSize,
"hopSize", hopSize);
Algorithm* windowing = factory.create("Windowing",
"type", "hann",
"normalized", false);
Algorithm* fft = factory.create("FFT", "size", frameSize);
frameCutter->input("signal").set(audio);
vector<Real> frame, windowedFrame;
vector<complex<Real>> fftFrame;
frameCutter->output("frame").set(frame);
windowing->input("frame").set(frame);
windowing->output("frame").set(windowedFrame);
fft->input("frame").set(windowedFrame);
fft->output("fft").set(fftFrame);
vector<vector<complex<Real>>> stftResult;
while (true) {
frameCutter->compute();
if (frame.empty()) break;
windowing->compute();
fft->compute();
stftResult.push_back(fftFrame);
}
🔁 实现 ISTFT
由于 Essentia 并未提供完整的 Overlap-Add 封装，我们需要仿照 librosa 逻辑手动实现。
Algorithm* ifft = factory.create("IFFT", "size", frameSize);
vector<Real> window = computeLibrosaHann(frameSize);
Real windowSum = 0.0;
for (int i = 0; i < hopSize; ++i) {
windowSum += window[i] * window[i];
}
const Real compensation = 1.0 / windowSum;
vector<Real> reconstructedAudio(originalLength, 0.0);
vector<Real> ifftOutputFrame(frameSize);
for (int i = 0; i < stftResult.size(); ++i) {
ifft->input("fft").set(stftResult[i]);
ifft->output("frame").set(ifftOutputFrame);
ifft->compute();
int pos = i * hopSize;
for (int n = 0; n < frameSize && pos + n < reconstructedAudio.size(); ++n) {
reconstructedAudio[pos + n] += ifftOutputFrame[n] * window[n] * compensation;
}
}
🧹 后处理与音频保存
removeDCOffset(reconstructedAudio);
conservativeNormalize(reconstructedAudio, 0.99);
// 保存音频
Algorithm* writer = factory.create("MonoWriter",
"filename", "./reconstructed.wav",
"sampleRate", 16000);
writer->input("audio").set(reconstructedAudio);
writer->compute();
delete writer;
🧪 代码完整示例
#include <iostream>
#include <essentia/essentia.h>
#include <essentia/algorithmfactory.h>
#include <essentia/pool.h>
#include <Eigen/Dense>
#include <unsupported/Eigen/CXX11/Tensor>
#include <numeric>
#include <cmath>
using namespace std;
using namespace essentia;
using namespace essentia::standard;
// Librosa风格的汉宁窗计算
vector<Real> computeLibrosaHann(int size) {
vector<Real> window(size);
for (int i = 0; i < size; ++i) {
window[i] = sin(M_PI * i / (size - 1)) * sin(M_PI * i / (size - 1));
}
return window;
}
// 移除直流偏移
void removeDCOffset(vector<Real>& audio) {
Real dcOffset = accumulate(audio.begin(), audio.end(), 0.0) / audio.size();
for (auto& sample : audio) {
sample -= dcOffset;
}
}
// 保守归一化
void conservativeNormalize(vector<Real>& audio, Real targetPeak = 0.99) {
Real peak = *max_element(audio.begin(), audio.end(),
[](Real a, Real b) { return abs(a) < abs(b); });
if (abs(peak) > 1e-6) { // 避免除以0
for (auto& sample : audio) {
sample *= (targetPeak / abs(peak));
}
}
}
int main() {
// 初始化Essentia
essentia::init();
AlgorithmFactory& factory = standard::AlgorithmFactory::instance();
// 1. 加载音频
Algorithm* loader = factory.create("MonoLoader",
"filename", "/work/000002.wav",
"sampleRate", 16000);
vector<Real> audio;
loader->output("audio").set(audio);
loader->compute();
delete loader;
cout << "Loaded audio with " << audio.size() << " samples" << endl;
// 2. STFT参数设置
const int frameSize = 320; // 与librosa默认值一致
const int hopSize = 80; // 25%重叠
const int originalLength = audio.size();
const string windowType = "hann";
// 3. STFT处理
Algorithm* frameCutter = factory.create("FrameCutter",
"frameSize", frameSize,
"hopSize", hopSize);
Algorithm* windowing = factory.create("Windowing",
"type", windowType,
"normalized", false,
"zeroPhase", false);
Algorithm* fft = factory.create("FFT", "size", frameSize);
frameCutter->input("signal").set(audio);
vector<Real> frame, windowedFrame;
frameCutter->output("frame").set(frame);
windowing->input("frame").set(frame);
windowing->output("frame").set(windowedFrame);
vector<complex<Real>> fftFrame;
fft->input("frame").set(windowedFrame);
fft->output("fft").set(fftFrame);
vector<vector<complex<Real>>> stftResult;
// STFT处理循环
while (true) {
frameCutter->compute();
if (frame.empty()) break;
windowing->compute();
fft->compute();
stftResult.push_back(fftFrame);
}
delete frameCutter;
delete windowing;
delete fft;
cout << "STFT completed. Frames: " << stftResult.size()
<< ", Bins: " << (stftResult.empty() ? 0 : stftResult[0].size()) << endl;
// 4. ISTFT处理（librosa风格）
Algorithm* ifft = factory.create("IFFT", "size", frameSize);
// 计算窗函数和补偿因子
vector<Real> window = computeLibrosaHann(frameSize);
Real windowSum = 0.0;
for (int i = 0; i < hopSize; ++i) {
windowSum += window[i] * window[i];
}
const Real compensation = 1.0 / windowSum;
// 重建音频初始化
vector<Real> reconstructedAudio(originalLength, 0.0);
vector<Real> ifftOutputFrame(frameSize);
// 处理每一帧
for (int i = 0; i < stftResult.size(); ++i) {
// IFFT变换
ifft->input("fft").set(stftResult[i]);
ifft->output("frame").set(ifftOutputFrame);
ifft->compute();
// 重叠相加（librosa风格）
int pos = i * hopSize;
for (int n = 0; n < frameSize && pos + n < reconstructedAudio.size(); ++n) {
reconstructedAudio[pos + n] += ifftOutputFrame[n] * window[n] * compensation;
}
}
delete ifft;
// 5. 后处理
removeDCOffset(reconstructedAudio);
conservativeNormalize(reconstructedAudio, 0.99);
// 6. 结果验证
// 计算RMS能量比
auto computeRMS = [](const vector<Real>& x) {
return sqrt(accumulate(x.begin(), x.end(), 0.0,
[](Real sum, Real val) { return sum + val*val; }) / x.size());
};
Real originalRMS = computeRMS(audio);
Real reconstructedRMS = computeRMS(reconstructedAudio);
cout << "Volume ratio (reconstructed/original): "
<< reconstructedRMS / originalRMS << endl;
// 7. 保存结果
Algorithm* writer = factory.create("MonoWriter",
"filename", "./reconstructed.wav",
"sampleRate", 16000);
writer->input("audio").set(reconstructedAudio);
writer->compute();
delete writer;
essentia::shutdown();
return 0;
}
🧩 CMake 配置示例
Essentia 依赖众多第三方库，下面是一个完整的 CMakeLists.txt 配置参考：
cmake_minimum_required(VERSION 3.10)
project(gcrn_cpp)
set(CMAKE_CXX_STANDARD 17)
file(GLOB_RECURSE CORE_SOURCE_FILES ${CMAKE_CURRENT_LIST_DIR}/source/*.cpp)
include_directories(${CMAKE_CURRENT_LIST_DIR}/3rdparty)
link_directories(${CMAKE_CURRENT_LIST_DIR}/libs/)
# 设置 PKG_CONFIG 路径
set(ENV{PKG_CONFIG_PATH} "/essentia-master/packaging/debian_3rdparty/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}")
find_package(PkgConfig REQUIRED)
# 依赖库查找
pkg_check_modules(SWRESAMPLE REQUIRED libswresample)
pkg_check_modules(AVCODEC REQUIRED libavcodec)
pkg_check_modules(AVFORMAT REQUIRED libavformat)
pkg_check_modules(SAMPLERATE REQUIRED samplerate)
pkg_check_modules(FFTW3 REQUIRED fftw3f)
pkg_check_modules(CHROMAPRINT REQUIRED libchromaprint)
pkg_check_modules(TAGLIB REQUIRED taglib)
pkg_check_modules(YAML REQUIRED yaml-0.1)
pkg_check_modules(EIGEN3 REQUIRED eigen3)
include_directories(${EIGEN3_INCLUDE_DIRS})
link_directories(${SAMPLERATE_LIBRARY_DIRS})
add_executable(gcrn_cpp main.cpp ${CORE_SOURCE_FILES})
target_link_libraries(gcrn_cpp PRIVATE
essentia
${CHROMAPRINT_LIBRARIES}
${SWRESAMPLE_LIBRARIES}
${SAMPLERATE_LIBRARIES}
${AVFORMAT_LIBRARIES}
${AVCODEC_LIBRARIES}
${AVUTIL_LIBRARIES}
${FFTW3_LIBRARIES}
${TAGLIB_LIBRARIES}
${YAML_LIBRARIES}
pthread dl m z
)
📚 参考资料
FAQ 页面：Frequently Asked Questions — Essentia 2.1-beta6-dev documentation
Librosa STFT 文档：https://librosa.org/doc/main/generated/librosa.stft.html