sound: implementing the fft chart

CMakeLists.txt

@@ -46,6 +46,7 @@ find_package(KF${QT_MAJOR_VERSION} REQUIRED COMPONENTS
 
 find_package(Threads REQUIRED)
 find_package(OpenCV REQUIRED)
+find_package(FFTW3 REQUIRED)
 find_package(PkgConfig REQUIRED)
 
 pkg_check_modules(LIBV4L2 libv4l2)

src/CMakeLists.txt

@@ -16,7 +16,12 @@ target_sources(eyeofsauron PRIVATE
  resources.qrc
 )
 
-target_include_directories(eyeofsauron PRIVATE ${OpenCV_INCLUDE_DIRS} ${LIBV4L2_INCLUDE_DIRS} ${LIBMEDIAINFO_INCLUDE_DIRS})
+target_include_directories(eyeofsauron PRIVATE 
+ ${OpenCV_INCLUDE_DIRS} 
+ ${FFTW3_INCLUDE_DIRS} 
+ ${LIBV4L2_INCLUDE_DIRS} 
+ ${LIBMEDIAINFO_INCLUDE_DIRS}
+)
 
 target_link_libraries(eyeofsauron PRIVATE
  Qt${QT_MAJOR_VERSION}::Core
@@ -32,6 +37,7 @@ target_link_libraries(eyeofsauron PRIVATE
  KF${QT_MAJOR_VERSION}::ConfigCore
  KF${QT_MAJOR_VERSION}::ConfigGui
  ${OpenCV_LIBS}
+ ${FFTW3_LIBRARIES}
  ${LIBV4L2_LIBRARIES}
  ${LIBMEDIAINFO_LIBRARIES}
 )

src/contents/ui/SoundWave.qml

@@ -45,6 +45,7 @@ Kirigami.ScrollablePage {
  Connections {
  function onUpdateChart() {
  EoSSoundBackend.updateSeriesWaveform(chartWaveForm.series(0));
+ EoSSoundBackend.updateSeriesFFT(chartFFT.series(0));
  }
 
  target: EoSSoundBackend
@@ -134,15 +135,24 @@ Kirigami.ScrollablePage {
  antialiasing: true
  theme: EoSdb.darkChartTheme === true ? ChartView.ChartThemeDark : ChartView.ChartThemeLight
  localizeNumbers: true
- title: i18n("Fourier Transform")
 
- ValueAxis {
- id: axisFrequency
+ LogValueAxis {
+ id: axisFreq
 
- labelFormat: "%.1f"
- // min: EoSSoundBackend.xAxisMin
- // max: EoSSoundBackend.xAxisMax
+ labelFormat: "%.1e"
+ min: EoSSoundBackend.xAxisMinFFT
+ max: EoSSoundBackend.xAxisMaxFFT
  titleText: i18n("Frequency [Hz]")
+ base: 10
+ }
+
+ ValueAxis {
+ id: axisFFT
+
+ labelFormat: "%.1e"
+ min: EoSSoundBackend.yAxisMinFFT
+ max: EoSSoundBackend.yAxisMaxFFT
+ titleText: i18n("Amplitude²")
  }
 
  Rectangle {
@@ -162,14 +172,23 @@ Kirigami.ScrollablePage {
  zoomRect: zoomRectFFT
  }
 
+ LineSeries {
+ id: chartFFTLineSeries
+
+ name: i18n("Fourier Transform")
+ axisX: axisFreq
+ axisY: axisFFT
+ useOpenGL: true
+ }
+
  }
 
  Text {
  Layout.fillWidth: true
  horizontalAlignment: Text.AlignHCenter
  color: Kirigami.Theme.textColor
  text: {
- return `x = ${mouseAreaFFT.mouseX.toFixed(1)} \t y = ${mouseAreaFFT.mouseY.toFixed(1)}`;
+ return `x = ${mouseAreaFFT.mouseX.toExponential(5)} \t y = ${mouseAreaFFT.mouseY.toExponential(5)}`;
  }
  }
 
@@ -182,9 +201,9 @@ Kirigami.ScrollablePage {
  footer: Kirigami.ActionToolBar {
  actions: [
  Kirigami.Action {
- text: i18n("Time Window")
 
  displayComponent: EoSSpinBox {
+ label: i18n("Time Window")
  unit: i18n("s")
  decimals: 2
  stepSize: 0.01

src/sound_wave.cpp

@@ -1,4 +1,5 @@
 #include "sound_wave.hpp"
+#include <fftw3.h>
 #include <qabstractseries.h>
 #include <qaudiodevice.h>
 #include <qaudioformat.h>
@@ -13,8 +14,10 @@
 #include <qxyseries.h>
 #include <QMediaDevices>
 #include <algorithm>
+#include <cmath>
 #include <memory>
 #include <mutex>
+#include <numbers>
 #include <vector>
 #include "config.h"
 #include "eyeofsauron_db.h"
@@ -165,6 +168,57 @@ void Backend::find_microphones() {
  }
 }
 
+void Backend::calc_fft() {
+ if (waveform.empty()) {
+ return;
+ }
+
+ fft_list.resize(waveform.size() / 2U + 1U);
+
+ real_input.resize(0);
+
+ for (const auto& p : waveform) {
+ real_input.emplace_back(p.y());
+ }
+
+ for (uint n = 0U; n < real_input.size(); n++) {
+ // https://en.wikipedia.org/wiki/Hann_function
+
+ const float w = 0.5F * (1.0F - std::cos(2.0F * std::numbers::pi_v<float> * static_cast<float>(n) /
+ static_cast<float>(real_input.size() - 1U)));
+
+ real_input[n] *= w;
+ }
+
+ auto* complex_output = fftw_alloc_complex(real_input.size());
+
+ auto* plan =
+ fftw_plan_dft_r2c_1d(static_cast<int>(real_input.size()), real_input.data(), complex_output, FFTW_ESTIMATE);
+
+ fftw_execute(plan);
+
+ for (uint i = 0U; i < fft_list.size(); i++) {
+ double sqr = complex_output[i][0] * complex_output[i][0] + complex_output[i][1] * complex_output[i][1];
+
+ sqr /= static_cast<double>(fft_list.size() * fft_list.size());
+
+ double f = 0.5F * static_cast<float>(microphone->format().sampleRate()) * static_cast<float>(i) /
+ static_cast<float>(fft_list.size());
+
+ fft_list[i] = QPointF(f, sqr);
+ }
+
+ // removing the DC component at f = 0 Hz
+
+ fft_list.erase(fft_list.begin());
+
+ if (complex_output != nullptr) {
+ fftw_free(complex_output);
+ }
+
+ fftw_destroy_plan(plan);
+}
+
 void Backend::process_buffer(const std::vector<double>& buffer) {
  double dt = 1.0 / microphone->format().sampleRate();
 
@@ -181,12 +235,19 @@ void Backend::process_buffer(const std::vector<double>& buffer) {
  waveform.removeFirst();
  }
 
- update_waveformt_chart_range();
+ calc_fft();
+
+ update_waveform_chart_range();
+ update_fft_chart_range();
 
  Q_EMIT updateChart();
 }
 
-void Backend::update_waveformt_chart_range() {
+void Backend::update_waveform_chart_range() {
+ if (waveform.empty()) {
+ return;
+ }
+
  auto [min_x, max_x] = std::ranges::minmax_element(waveform, [](QPointF a, QPointF b) { return a.x() < b.x(); });
  auto [min_y, max_y] = std::ranges::minmax_element(waveform, [](QPointF a, QPointF b) { return a.y() < b.y(); });
 
@@ -201,6 +262,25 @@ void Backend::update_waveformt_chart_range() {
  Q_EMIT yAxisMaxWaveChanged();
 }
 
+void Backend::update_fft_chart_range() {
+ if (fft_list.empty()) {
+ return;
+ }
+
+ auto [min_x, max_x] = std::ranges::minmax_element(fft_list, [](QPointF a, QPointF b) { return a.x() < b.x(); });
+ auto [min_y, max_y] = std::ranges::minmax_element(fft_list, [](QPointF a, QPointF b) { return a.y() < b.y(); });
+
+ _xAxisMinFFT = min_x->x();
+ _xAxisMaxFFT = max_x->x();
+ _yAxisMinFFT = min_y->y();
+ _yAxisMaxFFT = max_y->y();
+
+ Q_EMIT xAxisMinFFTChanged();
+ Q_EMIT xAxisMaxFFTChanged();
+ Q_EMIT yAxisMinFFTChanged();
+ Q_EMIT yAxisMaxFFTChanged();
+}
+
 void Backend::updateSeriesWaveform(QAbstractSeries* series) {
  if (series != nullptr) {
  auto xySeries = dynamic_cast<QXYSeries*>(series);
@@ -216,6 +296,21 @@ void Backend::updateSeriesWaveform(QAbstractSeries* series) {
  }
 }
 
+void Backend::updateSeriesFFT(QAbstractSeries* series) {
+ if (series != nullptr) {
+ auto xySeries = dynamic_cast<QXYSeries*>(series);
+
+ if (fft_list.empty()) {
+ return;
+ }
+
+ // Use replace instead of clear + append, it's optimized for performance
+ xySeries->replace(fft_list);
+ } else {
+ util::warning("series fft is null!");
+ }
+}
+
 void Backend::saveTable(const QUrl& fileUrl) {}
 
 void Backend::setPlayerPosition(qint64 value) {

src/sound_wave.hpp

@@ -59,6 +59,7 @@ class Backend : public QObject {
  Q_INVOKABLE void append(const QUrl& mediaUrl);
  Q_INVOKABLE void selectSource(const int& index);
  Q_INVOKABLE void updateSeriesWaveform(QAbstractSeries* series);
+ Q_INVOKABLE void updateSeriesFFT(QAbstractSeries* series);
  Q_INVOKABLE void saveTable(const QUrl& fileUrl);
  Q_INVOKABLE void setPlayerPosition(qint64 value);
 
@@ -108,10 +109,15 @@ class Backend : public QObject {
  std::mutex microphone_mutex;
 
  QList<QPointF> waveform;
+ QList<QPointF> fft_list;
+
+ std::vector<double> real_input;
 
  void find_microphones();
  void process_buffer(const std::vector<double>& buffer);
- void update_waveformt_chart_range();
+ void calc_fft();
+ void update_waveform_chart_range();
+ void update_fft_chart_range();
 };
 
 } // namespace sound