Skip to content

Commit

Permalink
implement mixer
Browse files Browse the repository at this point in the history
  • Loading branch information
@itsmattkc
itsmattkc committed Dec 12, 2024
1 parent 26aaa9b commit 41c9e54
Show file tree
Hide file tree
Showing 2 changed files with 146 additions and 66 deletions.
169 changes: 113 additions & 56 deletions app/viewer/mediapanel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ MediaPanel::MediaPanel(QWidget *parent) :
m_PlaybackTimer = new QTimer(this);
m_PlaybackTimer->setInterval(10);
connect(m_PlaybackTimer, &QTimer::timeout, this, &MediaPanel::TimerUpdate);

m_audioSink = nullptr;

m_audioDevice = new MediaAudioMixer(this);
m_audioDevice->SetMediaInstances(&m_mediaInstances);
}

MediaPanel::~MediaPanel()
Expand Down Expand Up @@ -371,15 +376,6 @@ void MediaPanel::OpenMediaInstance(si::Object *o)

void MediaPanel::Play(bool e)
{
{
// No matter what, stop any current audio
std::vector<QAudioSink*> copy = m_audioSinks;
for (auto it=copy.cbegin(); it!=copy.cend(); it++) {
auto o = *it;
o->stop();
}
}

if (e) {
bool has_video = false;
bool has_audio = false;
Expand All @@ -394,22 +390,23 @@ void MediaPanel::Play(bool e)
auto output_dev = QAudioDevice(QMediaDevices::defaultAudioOutput());
auto fmt = output_dev.preferredFormat();

ClearAudioSinks();

for (auto it=m_mediaInstances.cbegin(); it!=m_mediaInstances.cend(); it++) {
auto m = *it;
// Require float output (makes our lives easier)
fmt.setSampleFormat(QAudioFormat::Float);

for (size_t i = 0; i < m_mediaInstances.size(); i++) {
auto m = m_mediaInstances[i];

m->ResetEOF();

if (m->codec_type() == AVMEDIA_TYPE_VIDEO) {
has_video = true;
} else if (m->codec_type() == AVMEDIA_TYPE_AUDIO) {
if (m_PlaybackOffset < (m->GetDuration() + m->GetStartOffset())) {
if (m->StartPlayingAudio(output_dev, fmt)) {
auto out = new QAudioSink(output_dev, fmt, this);
out->setVolume(m->GetVolume());
out->start(m);
m_audioSinks.push_back(out);
if (m->SetUpResampleContext(fmt)) {
// auto out = new QAudioSink(output_dev, fmt, this);
// out->setVolume(m->GetVolume());
// out->start(m);
// m_audioSinks.push_back(out);
has_audio = true;
}
} else {
Expand All @@ -418,12 +415,28 @@ void MediaPanel::Play(bool e)
}
}

if (has_audio) {
m_audioDevice->SetAudioFormat(fmt);
m_audioDevice->open(QIODevice::ReadOnly);
m_audioDevice->SeekInSeconds(GetSecondsFromSlider());

m_audioSink = new QAudioSink(output_dev, fmt, this);
m_audioSink->start(m_audioDevice);
}

m_PlaybackStart = QDateTime::currentMSecsSinceEpoch();
m_PlaybackTimer->start();
m_PlayBtn->setText("Pause");
} else {
m_PlayBtn->setText("Play");
m_PlaybackTimer->stop();

if (m_audioSink) {
m_audioDevice->close();
m_audioSink->stop();
m_audioSink->deleteLater();
m_audioSink = nullptr;
}
}
m_PlayBtn->setChecked(e);
}
Expand Down Expand Up @@ -452,7 +465,6 @@ void MediaPanel::TimerUpdate()
}

if (all_eof) {
ClearAudioSinks();
Play(false);
m_PlayheadSlider->setValue(m_PlayheadSlider->maximum());
}
Expand Down Expand Up @@ -507,31 +519,6 @@ void MediaPanel::LabelContextMenuTriggered(const QPoint &pos)
m.exec(static_cast<QWidget*>(sender())->mapToGlobal(pos));
}

void MediaPanel::ClearAudioSinks()
{
if (m_audioSinks.size() != 0) {
for (auto s : m_audioSinks)
delete s;

m_audioSinks.clear();
}
}

qint64 MediaInstance::readData(char *data, qint64 maxSize)
{
return ReadAudio(data, maxSize);
}

qint64 MediaInstance::writeData(const char *data, qint64 maxSize)
{
return -1;
}

qint64 MediaInstance::size() const
{
return SecondsToBytes(m_startOffset + GetDuration());
}

ClickableSlider::ClickableSlider(Qt::Orientation orientation, QWidget *parent) :
QSlider(orientation, parent)
{
Expand All @@ -552,6 +539,7 @@ void ClickableSlider::mousePressEvent(QMouseEvent *e)
}

MediaInstance::MediaInstance(QObject *parent) :
QObject(parent),
m_FmtCtx(nullptr),
m_Packet(nullptr),
m_CodecCtx(nullptr),
Expand All @@ -562,7 +550,6 @@ MediaInstance::MediaInstance(QObject *parent) :
m_IoCtx(nullptr),
m_startOffset(0.0f)
{
this->open(QIODevice::ReadOnly);
}

void MediaInstance::Open(const si::bytearray &buf)
Expand Down Expand Up @@ -686,7 +673,7 @@ void MediaInstance::Close()
m_Data.Close();
}

bool MediaInstance::StartPlayingAudio(const QAudioDevice &output_dev, const QAudioFormat &fmt)
bool MediaInstance::SetUpResampleContext(const QAudioFormat &fmt)
{
if (m_SwrCtx) {
swr_free(&m_SwrCtx);
Expand Down Expand Up @@ -729,6 +716,8 @@ bool MediaInstance::StartPlayingAudio(const QAudioDevice &output_dev, const QAud
0, nullptr);
if (r < 0) {
qCritical() << "Failed to alloc swr ctx:" << r;
return false;
}
#else
m_SwrCtx = swr_alloc_set_opts(nullptr,
av_get_default_channel_layout(fmt.channelCount()),
Expand All @@ -740,19 +729,19 @@ bool MediaInstance::StartPlayingAudio(const QAudioDevice &output_dev, const QAud
0, nullptr);
if (!m_SwrCtx) {
qCritical() << "Failed to alloc swr ctx";
return false;
}
#endif
} else {
if (swr_init(m_SwrCtx) < 0) {
qCritical() << "Failed to init swr ctx";
} else {
m_AudioFlushed = false;
m_AudioBuffer.clear();

return true;
}
if (swr_init(m_SwrCtx) < 0) {
qCritical() << "Failed to init swr ctx";
return false;
}

return false;
m_AudioFlushed = false;
m_AudioBuffer.clear();

return true;
}

void MediaInstance::Seek(float seconds)
Expand Down Expand Up @@ -792,5 +781,73 @@ float MediaInstance::BytesToSeconds(int64_t t)
void MediaInstance::SetVirtualTime(float f)
{
m_virtualPosition = f - m_startOffset;
seek(SecondsToBytes(f));
}

MediaAudioMixer::MediaAudioMixer(QObject *parent) :
QIODevice(parent)
{
m_mediaInstances = nullptr;
}

void MediaAudioMixer::SeekInSeconds(float f)
{
seek(m_audioFormat.bytesForDuration(f * 1000000));
}

qint64 MediaAudioMixer::readData(char *data, qint64 maxSize)
{
if (!m_mediaInstances) {
return 0;
}

// Media instances should be set to same sample rate and channel count as output, but we may need to convert format
float *output = reinterpret_cast<float *>(data);

qint64 maxSamples = maxSize / m_audioFormat.bytesPerSample();

float *tmp = new float[maxSamples];

qint64 touchedBytes = 0;

for (auto it = m_mediaInstances->cbegin(); it != m_mediaInstances->cend(); it++) {
auto m = *it;

qint64 thisRead = m->ReadAudio(reinterpret_cast<char *>(tmp), maxSize);
if (thisRead > touchedBytes) {
memset(data + touchedBytes, 0, thisRead - touchedBytes);
touchedBytes = thisRead;
}

// TODO: Optimize with SSE and NEON
qint64 thisSamples = thisRead / m_audioFormat.bytesPerSample();
for (qint64 j = 0; j < thisSamples; j++) {
output[j] += tmp[j] * m->GetVolume();
}
}

delete [] tmp;

return touchedBytes;
}

qint64 MediaAudioMixer::writeData(const char *data, qint64 maxSize)
{
return -1;
}

qint64 MediaAudioMixer::size() const
{
if (!m_mediaInstances) {
return 0;
}

// Calculate maximum duration in seconds
float maxLength = 0;
for (auto it = m_mediaInstances->cbegin(); it != m_mediaInstances->cend(); it++) {
auto m = *it;
maxLength = qMax(maxLength, m->GetDuration() + m->GetStartOffset());
}

// Convert seconds to bytes in the output format
return m_audioFormat.bytesForDuration(maxLength * 1000000);
}
43 changes: 33 additions & 10 deletions app/viewer/mediapanel.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ extern "C" {
#include <QTimer>
#include "panel.h"

class MediaInstance : public QIODevice
class MediaInstance : public QObject
{
Q_OBJECT
public:
Expand All @@ -36,7 +36,7 @@ class MediaInstance : public QIODevice
return m_Stream ? m_Stream->codecpar->codec_type : AVMEDIA_TYPE_UNKNOWN;
}

bool StartPlayingAudio(const QAudioDevice &output_dev, const QAudioFormat &fmt);
bool SetUpResampleContext(const QAudioFormat &fmt);

void Seek(float seconds);

Expand Down Expand Up @@ -82,11 +82,6 @@ class MediaInstance : public QIODevice
signals:
void EndOfFile();

protected:
virtual qint64 readData(char *data, qint64 maxSize) override;
virtual qint64 writeData(const char *data, qint64 maxSize) override;
virtual qint64 size() const override;

private:
void ClearQueue();

Expand Down Expand Up @@ -124,6 +119,34 @@ class MediaInstance : public QIODevice

};

class MediaAudioMixer : public QIODevice
{
Q_OBJECT
public:
MediaAudioMixer(QObject *parent = nullptr);

void SetMediaInstances(std::vector<MediaInstance *> *mi)
{
m_mediaInstances = mi;
}

void SetAudioFormat(const QAudioFormat &fmt)
{
m_audioFormat = fmt;
}

void SeekInSeconds(float f);

protected:
virtual qint64 readData(char *data, qint64 maxSize) override;
virtual qint64 writeData(const char *data, qint64 maxSize) override;
virtual qint64 size() const override;

private:
std::vector<MediaInstance *> *m_mediaInstances;
QAudioFormat m_audioFormat;
};

class MediaPanel : public Panel
{
Q_OBJECT
Expand Down Expand Up @@ -159,7 +182,9 @@ class MediaPanel : public Panel

std::vector<QLabel *> m_imgViewers;
std::vector<MediaInstance *> m_mediaInstances;
std::vector<QAudioSink *> m_audioSinks;

QAudioSink *m_audioSink;
MediaAudioMixer *m_audioDevice;

QSlider *m_PlayheadSlider;
QPushButton *m_PlayBtn;
Expand All @@ -181,8 +206,6 @@ private slots:

void LabelContextMenuTriggered(const QPoint &pos);

void ClearAudioSinks();

};

class ClickableSlider : public QSlider
Expand Down

0 comments on commit 41c9e54

Please sign in to comment.