From 3b5245f9748b5afc08d5b0373329a92745b625f0 Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:03:23 -0800 Subject: [PATCH 01/18] Add basic multimedia container and encoder interfaces Adds the media subdirectory. --- src/CMakeLists.txt | 1 + src/media/CMakeLists.txt | 7 +++++ src/media/audio_encoder.hpp | 39 +++++++++++++++++++++++ src/media/container.hpp | 53 +++++++++++++++++++++++++++++++ src/media/encoder.hpp | 51 ++++++++++++++++++++++++++++++ src/media/video_encoder.hpp | 58 ++++++++++++++++++++++++++++++++++ src/media/video_frame.hpp | 63 +++++++++++++++++++++++++++++++++++++ 7 files changed, 272 insertions(+) create mode 100644 src/media/CMakeLists.txt create mode 100644 src/media/audio_encoder.hpp create mode 100644 src/media/container.hpp create mode 100644 src/media/encoder.hpp create mode 100644 src/media/video_encoder.hpp create mode 100644 src/media/video_frame.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8ff8942d5..ada3a3f4f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -549,6 +549,7 @@ if(SRB2_CONFIG_ENABLE_TESTS) add_subdirectory(tests) endif() add_subdirectory(menus) +add_subdirectory(media) # strip debug symbols into separate file when using gcc. # to be consistent with Makefile, don't generate for OS X. diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt new file mode 100644 index 000000000..f0a7575bd --- /dev/null +++ b/src/media/CMakeLists.txt @@ -0,0 +1,7 @@ +target_sources(SRB2SDL2 PRIVATE + audio_encoder.hpp + container.hpp + encoder.hpp + video_encoder.hpp + video_frame.hpp +) diff --git a/src/media/audio_encoder.hpp b/src/media/audio_encoder.hpp new file mode 100644 index 000000000..3aa9f5cf5 --- /dev/null +++ b/src/media/audio_encoder.hpp @@ -0,0 +1,39 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_AUDIO_ENCODER_HPP__ +#define __SRB2_MEDIA_AUDIO_ENCODER_HPP__ + +#include + +#include "encoder.hpp" + +namespace srb2::media +{ + +class AudioEncoder : virtual public MediaEncoder +{ +public: + using sample_buffer_t = tcb::span; + + struct Config + { + int channels; + int sample_rate; + }; + + virtual void encode(sample_buffer_t samples) = 0; + + virtual int channels() const = 0; + virtual int sample_rate() const = 0; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_AUDIO_ENCODER_HPP__ diff --git a/src/media/container.hpp b/src/media/container.hpp new file mode 100644 index 000000000..590d496f7 --- /dev/null +++ b/src/media/container.hpp @@ -0,0 +1,53 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_CONTAINER_HPP__ +#define __SRB2_MEDIA_CONTAINER_HPP__ + +#include +#include +#include +#include + +#include "audio_encoder.hpp" +#include "video_encoder.hpp" + +namespace srb2::media +{ + +class MediaContainer +{ +public: + using dtor_cb_t = std::function; + using time_unit_t = std::chrono::duration; + + struct Config + { + std::string file_name; + dtor_cb_t destructor_callback; + }; + + virtual ~MediaContainer() = default; + + virtual std::unique_ptr make_audio_encoder(AudioEncoder::Config config) = 0; + virtual std::unique_ptr make_video_encoder(VideoEncoder::Config config) = 0; + + virtual const char* name() const = 0; + virtual const char* file_name() const = 0; + + // These are normally estimates. However, when called from + // Config::destructor_callback, these are the exact final + // values. + virtual time_unit_t duration() const = 0; + virtual std::size_t size() const = 0; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_CONTAINER_HPP__ diff --git a/src/media/encoder.hpp b/src/media/encoder.hpp new file mode 100644 index 000000000..d6fa9c049 --- /dev/null +++ b/src/media/encoder.hpp @@ -0,0 +1,51 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_ENCODER_HPP__ +#define __SRB2_MEDIA_ENCODER_HPP__ + +#include +#include + +#include + +namespace srb2::media +{ + +class MediaEncoder +{ +public: + using time_unit_t = std::chrono::duration; + + struct BitRate + { + std::size_t bits; // 8 bits = 1 byte :) + time_unit_t period; + }; + + virtual ~MediaEncoder() = default; + + // Should be called finally but it's optional. + virtual void flush() = 0; + + virtual const char* name() const = 0; + + // Returns an average bit rate over a constant period of + // time, assuming no frames drops. + virtual BitRate estimated_bit_rate() const = 0; + +protected: + using frame_buffer_t = tcb::span; + + virtual void write_frame(frame_buffer_t frame, time_unit_t timestamp, bool is_key_frame) = 0; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_ENCODER_HPP__ diff --git a/src/media/video_encoder.hpp b/src/media/video_encoder.hpp new file mode 100644 index 000000000..1230bd8da --- /dev/null +++ b/src/media/video_encoder.hpp @@ -0,0 +1,58 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_VIDEO_ENCODER_HPP__ +#define __SRB2_MEDIA_VIDEO_ENCODER_HPP__ + +#include "encoder.hpp" +#include "video_frame.hpp" + +namespace srb2::media +{ + +class VideoEncoder : virtual public MediaEncoder +{ +public: + struct Config + { + int width; + int height; + int frame_rate; + VideoFrame::BufferMethod buffer_method; + }; + + struct FrameCount + { + // Number of real frames, not counting frame skips. + int frames; + + time_unit_t duration; + }; + + // VideoFrame::width() and VideoFrame::height() should be + // used on the returned frame. + virtual VideoFrame::instance_t new_frame(int width, int height, int pts) = 0; + + virtual void encode(VideoFrame::instance_t frame) = 0; + + virtual int width() const = 0; + virtual int height() const = 0; + virtual int frame_rate() const = 0; + + // Reports the number of threads used, if the encoder is + // multithreaded. + virtual int thread_count() const = 0; + + // Number of frames fully encoded so far. + virtual FrameCount frame_count() const = 0; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_VIDEO_ENCODER_HPP__ diff --git a/src/media/video_frame.hpp b/src/media/video_frame.hpp new file mode 100644 index 000000000..acf278ff5 --- /dev/null +++ b/src/media/video_frame.hpp @@ -0,0 +1,63 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_VIDEO_FRAME_HPP__ +#define __SRB2_MEDIA_VIDEO_FRAME_HPP__ + +#include +#include +#include + +#include + +namespace srb2::media +{ + +class VideoFrame +{ +public: + using instance_t = std::unique_ptr; + + enum class BufferMethod + { + // Returns an already allocated buffer for each + // frame. See VideoFrame::rgba_buffer(). The encoder + // completely manages allocating this buffer. + kEncoderAllocatedRGBA8888, + }; + + struct Buffer + { + tcb::span plane; + std::size_t row_stride; // size of each row + }; + + virtual int width() const = 0; + virtual int height() const = 0; + + int pts() const { return pts_; } + + // Returns a buffer that should be + // filled with RGBA pixels. + // + // This method may only be used if + // the encoder was configured with + // BufferMethod::kEncoderAllocatedRGBA8888. + virtual const Buffer& rgba_buffer() const = 0; + +protected: + VideoFrame(int pts) : pts_(pts) {} + +private: + int pts_; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_VIDEO_FRAME_HPP__ From e9f5a75d4a8d85db334dff9e72c8bb16393fff5f Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:05:25 -0800 Subject: [PATCH 02/18] media: add "options" cvar abstraction --- src/media/CMakeLists.txt | 2 + src/media/options.cpp | 118 +++++++++++++++++++++++++++++++++++++++ src/media/options.hpp | 51 +++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 src/media/options.cpp create mode 100644 src/media/options.hpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index f0a7575bd..fc4c6202a 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -2,6 +2,8 @@ target_sources(SRB2SDL2 PRIVATE audio_encoder.hpp container.hpp encoder.hpp + options.cpp + options.hpp video_encoder.hpp video_frame.hpp ) diff --git a/src/media/options.cpp b/src/media/options.cpp new file mode 100644 index 000000000..b933d9ad0 --- /dev/null +++ b/src/media/options.cpp @@ -0,0 +1,118 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include + +#include + +#include "../cxxutil.hpp" +#include "../m_fixed.h" +#include "options.hpp" + +using namespace srb2::media; + +static std::vector g_cvars; + +Options::Options(const char* prefix, map_t map) : prefix_(prefix), map_(map) +{ + for (auto& [suffix, cvar] : map_) + { + cvar.name = strdup(fmt::format("{}_{}", prefix_, suffix).c_str()); + g_cvars.emplace_back(&cvar); + } +} + +const consvar_t& Options::cvar(const char* option) const +{ + const consvar_t& cvar = map_.at(option); + + SRB2_ASSERT(cvar.string != nullptr); + + return cvar; +} + +template <> +int Options::get(const char* option) const +{ + return cvar(option).value; +} + +template <> +float Options::get(const char* option) const +{ + return FixedToFloat(cvar(option).value); +} + +static consvar_t range_cvar(const char* default_value, int32_t min, int32_t max, int32_t flags = 0) +{ + return CVAR_INIT( + nullptr, + default_value, + CV_SAVE | flags, + new CV_PossibleValue_t[] {{min, "MIN"}, {max, "MAX"}, {}}, + nullptr + ); +} + +template <> +consvar_t Options::range(const char* default_value, float min, float max) +{ + return range_cvar(default_value, FloatToFixed(min), FloatToFixed(max), CV_FLOAT); +} + +template <> +consvar_t Options::range_min(const char* default_value, float min) +{ + return range_cvar(default_value, FloatToFixed(min), INT32_MAX); +} + +template <> +consvar_t Options::range(const char* default_value, int min, int max) +{ + return range_cvar(default_value, min, max); +} + +template <> +consvar_t Options::range_min(const char* default_value, int min) +{ + return range_cvar(default_value, min, INT32_MAX); +} + +template <> +consvar_t Options::value_map(const char* default_value, std::map values) +{ + auto* arr = new CV_PossibleValue_t[values.size() + 1]; + + std::size_t i = 0; + + for (const auto& [k, v] : values) + { + arr[i].value = v; + arr[i].strvalue = k; + + i++; + } + + arr[i].value = 0; + arr[i].strvalue = nullptr; + + return CVAR_INIT(nullptr, default_value, CV_SAVE, arr, nullptr); +} + +void srb2::media::register_options() +{ + for (auto cvar : g_cvars) + { + CV_RegisterVar(cvar); + } + + g_cvars = {}; +} diff --git a/src/media/options.hpp b/src/media/options.hpp new file mode 100644 index 000000000..d7ce7fd2a --- /dev/null +++ b/src/media/options.hpp @@ -0,0 +1,51 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_OPTIONS_HPP__ +#define __SRB2_MEDIA_OPTIONS_HPP__ + +#include +#include + +#include "../command.h" + +namespace srb2::media +{ + +class Options +{ +public: + using map_t = std::unordered_map; + + Options(const char* prefix, map_t map); + + template + T get(const char* option) const; + + template + static consvar_t range(const char* default_value, T min, T max); + + template + static consvar_t range_min(const char* default_value, T min); + + template + static consvar_t value_map(const char* default_value, std::map values); + +private: + const char* prefix_; + map_t map_; + + const consvar_t& cvar(const char* option) const; +}; + +void register_options(); + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_OPTIONS_HPP__ From 650264ea86f39d0e1232046babcd6e0004254d6b Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:07:45 -0800 Subject: [PATCH 03/18] media: add libvorbis encoder --- src/media/CMakeLists.txt | 3 + src/media/vorbis.cpp | 138 +++++++++++++++++++++++++++++++++++++ src/media/vorbis.hpp | 54 +++++++++++++++ src/media/vorbis_error.hpp | 54 +++++++++++++++ 4 files changed, 249 insertions(+) create mode 100644 src/media/vorbis.cpp create mode 100644 src/media/vorbis.hpp create mode 100644 src/media/vorbis_error.hpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index fc4c6202a..1ecf75d21 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -6,4 +6,7 @@ target_sources(SRB2SDL2 PRIVATE options.hpp video_encoder.hpp video_frame.hpp + vorbis.cpp + vorbis.hpp + vorbis_error.hpp ) diff --git a/src/media/vorbis.cpp b/src/media/vorbis.cpp new file mode 100644 index 000000000..6b33aa22e --- /dev/null +++ b/src/media/vorbis.cpp @@ -0,0 +1,138 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include + +#include +#include + +#include "../cxxutil.hpp" +#include "vorbis.hpp" +#include "vorbis_error.hpp" + +using namespace srb2::media; + +// clang-format off +const Options VorbisEncoder::options_("vorbis", { + {"quality", Options::range("0", -0.1f, 1.f)}, + {"max_bitrate", Options::range_min("-1", -1)}, + {"nominal_bitrate", Options::range_min("-1", -1)}, + {"min_bitrate", Options::range_min("-1", -1)}, +}); +// clang-format on + +VorbisEncoder::VorbisEncoder(Config cfg) +{ + const long max_bitrate = options_.get("max_bitrate"); + const long nominal_bitrate = options_.get("nominal_bitrate"); + const long min_bitrate = options_.get("min_bitrate"); + + vorbis_info_init(&vi_); + + if (max_bitrate != -1 || nominal_bitrate != -1 || min_bitrate != -1) + { + // managed bitrate mode + VorbisError error = + vorbis_encode_init(&vi_, cfg.channels, cfg.sample_rate, max_bitrate, nominal_bitrate, min_bitrate); + + if (error != 0) + { + throw std::invalid_argument(fmt::format( + "vorbis_encode_init: {}, max_bitrate={}, nominal_bitrate={}, min_bitrate={}", + error, + max_bitrate, + nominal_bitrate, + min_bitrate + )); + } + } + else + { + // variable bitrate mode + const float quality = options_.get("quality"); + + VorbisError error = vorbis_encode_init_vbr(&vi_, cfg.channels, cfg.sample_rate, quality); + + if (error != 0) + { + throw std::invalid_argument(fmt::format("vorbis_encode_init: {}, quality={}", error, quality)); + } + } + + SRB2_ASSERT(vorbis_analysis_init(&vd_, &vi_) == 0); + SRB2_ASSERT(vorbis_block_init(&vd_, &vb_) == 0); +} + +VorbisEncoder::~VorbisEncoder() +{ + vorbis_block_clear(&vb_); + vorbis_dsp_clear(&vd_); + vorbis_info_clear(&vi_); +} + +VorbisEncoder::headers_t VorbisEncoder::generate_headers() +{ + headers_t op; + + vorbis_comment vc; + vorbis_comment_init(&vc); + + VorbisError error = vorbis_analysis_headerout(&vd_, &vc, &op[0], &op[1], &op[2]); + + if (error != 0) + { + throw std::invalid_argument(fmt::format("vorbis_analysis_headerout: {}", error)); + } + + vorbis_comment_clear(&vc); + + return op; +} + +void VorbisEncoder::analyse(sample_buffer_t in) +{ + const int ch = channels(); + + const std::size_t n = in.size() / ch; + float** fv = vorbis_analysis_buffer(&vd_, n); + + for (std::size_t i = 0; i < n; ++i) + { + auto s = in.subspan(i * ch, ch); + + fv[0][i] = s[0]; + fv[1][i] = s[1]; + } + + // automatically handles end of stream if n = 0 + SRB2_ASSERT(vorbis_analysis_wrote(&vd_, n) == 0); + + while (vorbis_analysis_blockout(&vd_, &vb_) > 0) + { + SRB2_ASSERT(vorbis_analysis(&vb_, nullptr) == 0); + SRB2_ASSERT(vorbis_bitrate_addblock(&vb_) == 0); + + ogg_packet op; + + while (vorbis_bitrate_flushpacket(&vd_, &op) > 0) + { + write_packet(&op); + } + } +} + +void VorbisEncoder::write_packet(ogg_packet* op) +{ + using T = const std::byte; + tcb::span p(reinterpret_cast(op->packet), static_cast(op->bytes)); + + write_frame(p, std::chrono::duration(vorbis_granule_time(&vd_, op->granulepos)), true); +} diff --git a/src/media/vorbis.hpp b/src/media/vorbis.hpp new file mode 100644 index 000000000..333f31f8e --- /dev/null +++ b/src/media/vorbis.hpp @@ -0,0 +1,54 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_VORBIS_HPP__ +#define __SRB2_MEDIA_VORBIS_HPP__ + +#include + +#include + +#include "audio_encoder.hpp" +#include "options.hpp" + +namespace srb2::media +{ + +class VorbisEncoder : public AudioEncoder +{ +public: + static const Options options_; + + VorbisEncoder(Config config); + ~VorbisEncoder(); + + virtual void encode(sample_buffer_t samples) override final { analyse(samples); } + virtual void flush() override final { analyse(); } + + virtual const char* name() const override final { return "Vorbis"; } + virtual int channels() const override final { return vi_.channels; } + virtual int sample_rate() const override final { return vi_.rate; } + +protected: + using headers_t = std::array; + + headers_t generate_headers(); + +private: + vorbis_info vi_; + vorbis_dsp_state vd_; + vorbis_block vb_; + + void analyse(sample_buffer_t samples = {}); + void write_packet(ogg_packet* op); +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_VORBIS_HPP__ diff --git a/src/media/vorbis_error.hpp b/src/media/vorbis_error.hpp new file mode 100644 index 000000000..7c4f9d3d6 --- /dev/null +++ b/src/media/vorbis_error.hpp @@ -0,0 +1,54 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_VORBIS_ERROR_HPP__ +#define __SRB2_MEDIA_VORBIS_ERROR_HPP__ + +#include + +#include +#include + +class VorbisError +{ +public: + VorbisError(int error) : error_(error) {} + + operator int() const { return error_; } + + std::string name() const + { + switch (error_) + { + case OV_EFAULT: + return "Internal error (OV_EFAULT)"; + case OV_EINVAL: + return "Invalid settings (OV_EINVAL)"; + case OV_EIMPL: + return "Invalid settings (OV_EIMPL)"; + default: + return fmt::format("error {}", error_); + } + } + +private: + int error_; +}; + +template <> +struct fmt::formatter : formatter +{ + template + auto format(const VorbisError& error, FormatContext& ctx) const + { + return formatter::format(error.name(), ctx); + } +}; + +#endif // __SRB2_MEDIA_VORBIS_ERROR_HPP__ From 14152541316a28a39a5f855294f09f4ce3cbf0f5 Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:09:35 -0800 Subject: [PATCH 04/18] media: add YUV420p module Converts RGBA image to YUV420p, useful for most video codecs. --- src/media/CMakeLists.txt | 2 + src/media/yuv420p.cpp | 124 +++++++++++++++++++++++++++++++++++++++ src/media/yuv420p.hpp | 74 +++++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 src/media/yuv420p.cpp create mode 100644 src/media/yuv420p.hpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index 1ecf75d21..d0ade1566 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -9,4 +9,6 @@ target_sources(SRB2SDL2 PRIVATE vorbis.cpp vorbis.hpp vorbis_error.hpp + yuv420p.cpp + yuv420p.hpp ) diff --git a/src/media/yuv420p.cpp b/src/media/yuv420p.cpp new file mode 100644 index 000000000..bfc0fd7c5 --- /dev/null +++ b/src/media/yuv420p.cpp @@ -0,0 +1,124 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include + +#include +#include +#include + +#include "../cxxutil.hpp" +#include "yuv420p.hpp" + +using namespace srb2::media; + +bool YUV420pFrame::BufferRGBA::resize(int width, int height) +{ + if (width == width_ && height == height_) + { + return false; + } + + width_ = width; + height_ = height; + + row_stride = width * 4; + + const std::size_t new_size = row_stride * height; + + // Overallocate since the vector's alignment can't be + // easily controlled. This is not a significant waste. + vec_.resize(new_size + (kAlignment - 1)); + + void* p = vec_.data(); + std::size_t n = vec_.size(); + + SRB2_ASSERT(std::align(kAlignment, 1, p, n) != nullptr); + + plane = tcb::span(reinterpret_cast(p), new_size); + + return true; +} + +void YUV420pFrame::BufferRGBA::erase() +{ + std::fill(vec_.begin(), vec_.end(), 0); +} + +void YUV420pFrame::BufferRGBA::release() +{ + if (!vec_.empty()) + { + *this = {}; + } +} + +const VideoFrame::Buffer& YUV420pFrame::rgba_buffer() const +{ + return *rgba_; +} + +void YUV420pFrame::convert() const +{ + // ABGR = RGBA in memory + libyuv::ABGRToI420( + rgba_->plane.data(), + rgba_->row_stride, + y_.plane.data(), + y_.row_stride, + u_.plane.data(), + u_.row_stride, + v_.plane.data(), + v_.row_stride, + width(), + height() + ); +} + +void YUV420pFrame::scale(const BufferRGBA& scaled_rgba) +{ + int vw = scaled_rgba.width(); + int vh = scaled_rgba.height(); + + uint8_t* p = scaled_rgba.plane.data(); + + const float ru = width() / static_cast(height()); + const float rs = vw / static_cast(vh); + + // Maintain aspect ratio of unscaled. Fit inside scaled + // aspect by centering image. + + if (rs > ru) // scaled is wider + { + vw = vh * ru; + p += (scaled_rgba.width() - vw) / 2 * 4; + } + else + { + vh = vw / ru; + p += (scaled_rgba.height() - vh) / 2 * scaled_rgba.row_stride; + } + + // Curiously, this function doesn't care about channel order. + libyuv::ARGBScale( + rgba_->plane.data(), + rgba_->row_stride, + width(), + height(), + p, + scaled_rgba.row_stride, + vw, + vh, + libyuv::FilterMode::kFilterNone + ); + + rgba_ = &scaled_rgba; +} diff --git a/src/media/yuv420p.hpp b/src/media/yuv420p.hpp new file mode 100644 index 000000000..f8be45e06 --- /dev/null +++ b/src/media/yuv420p.hpp @@ -0,0 +1,74 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_YUV420P_HPP__ +#define __SRB2_MEDIA_YUV420P_HPP__ + +#include +#include + +#include "video_frame.hpp" + +namespace srb2::media +{ + +class YUV420pFrame : public VideoFrame +{ +public: + // 32-byte aligned for AVX optimizations (see libyuv) + static constexpr int kAlignment = 32; + + class BufferRGBA : public VideoFrame::Buffer + { + public: + bool resize(int width, int height); // true if resized + + void erase(); // fills with black + void release(); + + int width() const { return width_; } + int height() const { return height_; } + + private: + int width_, height_; + + std::vector vec_; + }; + + YUV420pFrame(int pts, Buffer y, Buffer u, Buffer v, const BufferRGBA& rgba) : + VideoFrame(pts), y_(y), u_(u), v_(v), rgba_(&rgba) + { + } + + ~YUV420pFrame() = default; + + // Simply resets PTS and RGBA buffer while keeping YUV + // buffers intact. + void reset(int pts, const BufferRGBA& rgba) { *this = YUV420pFrame(pts, y_, u_, v_, rgba); } + + // Converts RGBA buffer to YUV planes. + void convert() const; + + // Scales the existing buffer into a new one. This new + // buffer replaces the existing one. + void scale(const BufferRGBA& rgba); + + virtual int width() const override { return rgba_->width(); } + virtual int height() const override { return rgba_->height(); } + + virtual const Buffer& rgba_buffer() const override; + +private: + Buffer y_, u_, v_; + const BufferRGBA* rgba_; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_YUV420P_HPP__ From b8015b4ad2fd2a3f296f507f5287ff662351d7eb Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:08:38 -0800 Subject: [PATCH 05/18] media: add libvpx VP8 encoder --- src/media/CMakeLists.txt | 3 + src/media/vp8.cpp | 293 +++++++++++++++++++++++++++++++++++++++ src/media/vp8.hpp | 102 ++++++++++++++ src/media/vpx_error.hpp | 45 ++++++ 4 files changed, 443 insertions(+) create mode 100644 src/media/vp8.cpp create mode 100644 src/media/vp8.hpp create mode 100644 src/media/vpx_error.hpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index d0ade1566..3e748c08a 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -9,6 +9,9 @@ target_sources(SRB2SDL2 PRIVATE vorbis.cpp vorbis.hpp vorbis_error.hpp + vp8.cpp + vp8.hpp + vpx_error.hpp yuv420p.cpp yuv420p.hpp ) diff --git a/src/media/vp8.cpp b/src/media/vp8.cpp new file mode 100644 index 000000000..341b1b932 --- /dev/null +++ b/src/media/vp8.cpp @@ -0,0 +1,293 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "../cxxutil.hpp" +#include "vp8.hpp" +#include "vpx_error.hpp" +#include "yuv420p.hpp" + +using namespace srb2::media; + +namespace +{ + +namespace KeyFrameOption +{ + +enum : int +{ + kAuto = -1, +}; + +}; // namespace KeyFrameOption + +namespace DeadlineOption +{ + +enum : int +{ + kInfinite = 0, +}; + +}; // namespace DeadlineOption + +}; // namespace + +// clang-format off +const Options VP8Encoder::options_("vp8", { + {"quality_mode", Options::value_map("q", { + {"vbr", VPX_VBR}, + {"cbr", VPX_CBR}, + {"cq", VPX_CQ}, + {"q", VPX_Q}, + })}, + {"target_bitrate", Options::range_min("800", 1)}, + {"min_q", Options::range("4", 4, 63)}, + {"max_q", Options::range("55", 4, 63)}, + {"kf_min", Options::range_min("0", 0)}, + {"kf_max", Options::value_map("auto", { + {"auto", KeyFrameOption::kAuto}, + {"MIN", 0}, + {"MAX", INT32_MAX}, + })}, + {"cpu_used", Options::range("0", -16, 16)}, + {"cq_level", Options::range("10", 0, 63)}, + {"deadline", Options::value_map("10", { + {"infinite", DeadlineOption::kInfinite}, + {"MIN", 1}, + {"MAX", INT32_MAX}, + })}, + {"sharpness", Options::range("7", 0, 7)}, + {"token_parts", Options::range("0", 0, 3)}, + {"threads", Options::range_min("1", 1)}, +}); +// clang-format on + +vpx_codec_iface_t* VP8Encoder::kCodec = vpx_codec_vp8_cx(); + +const vpx_codec_enc_cfg_t VP8Encoder::configure(const Config user) +{ + vpx_codec_enc_cfg_t cfg; + vpx_codec_enc_config_default(kCodec, &cfg, 0); + + cfg.g_threads = options_.get("threads"); + + cfg.g_w = user.width; + cfg.g_h = user.height; + + cfg.g_bit_depth = VPX_BITS_8; + cfg.g_input_bit_depth = 8; + + cfg.g_timebase.num = 1; + cfg.g_timebase.den = user.frame_rate; + + cfg.g_pass = VPX_RC_ONE_PASS; + cfg.rc_end_usage = static_cast(options_.get("quality_mode")); + cfg.kf_mode = VPX_KF_AUTO; + + cfg.rc_target_bitrate = options_.get("target_bitrate"); + cfg.rc_min_quantizer = options_.get("min_q"); + cfg.rc_max_quantizer = options_.get("max_q"); + + // Keyframe spacing, in number of frames. + // kf_max_dist should be low enough to allow scrubbing. + + int kf_max = options_.get("kf_max"); + + if (kf_max == KeyFrameOption::kAuto) + { + // Automatically pick a good rate + kf_max = (user.frame_rate / 2); // every .5s + } + + cfg.kf_min_dist = options_.get("kf_min"); + cfg.kf_max_dist = kf_max; + + return cfg; +} + +VP8Encoder::VP8Encoder(Config config) : ctx_(config), img_(config.width, config.height), frame_rate_(config.frame_rate) +{ + SRB2_ASSERT(config.buffer_method == VideoFrame::BufferMethod::kEncoderAllocatedRGBA8888); + + control(VP8E_SET_CPUUSED, "cpu_used"); + control(VP8E_SET_CQ_LEVEL, "cq_level"); + control(VP8E_SET_SHARPNESS, "sharpness"); + control(VP8E_SET_TOKEN_PARTITIONS, "token_parts"); + + auto plane = [this](int k, int ycs = 0) + { + using T = uint8_t; + auto view = tcb::span(reinterpret_cast(img_->planes[k]), img_->stride[k] * (img_->h >> ycs)); + + return VideoFrame::Buffer {view, static_cast(img_->stride[k])}; + }; + + frame_ = std::make_unique( + 0, + plane(VPX_PLANE_Y), + plane(VPX_PLANE_U, img_->y_chroma_shift), + plane(VPX_PLANE_V, img_->y_chroma_shift), + rgba_buffer_ + ); +} + +VP8Encoder::CtxWrapper::CtxWrapper(const Config user) +{ + const vpx_codec_enc_cfg_t cfg = configure(user); + + if (vpx_codec_enc_init(&ctx_, kCodec, &cfg, 0) != VPX_CODEC_OK) + { + throw std::invalid_argument(fmt::format("vpx_codec_enc_init: {}", VpxError(ctx_))); + } +} + +VP8Encoder::CtxWrapper::~CtxWrapper() +{ + vpx_codec_destroy(&ctx_); +} + +VP8Encoder::ImgWrapper::ImgWrapper(int width, int height) +{ + SRB2_ASSERT(vpx_img_alloc(&img_, VPX_IMG_FMT_I420, width, height, YUV420pFrame::kAlignment) != nullptr); +} + +VP8Encoder::ImgWrapper::~ImgWrapper() +{ + vpx_img_free(&img_); +} + +VideoFrame::instance_t VP8Encoder::new_frame(int width, int height, int pts) +{ + SRB2_ASSERT(frame_ != nullptr); + + if (rgba_buffer_.resize(width, height)) + { + // If there was a resize, the aspect ratio may not + // match. When the frame is scaled later, it will be + // "fit" into the target aspect ratio, leaving some + // empty space around the scaled image. (See + // VP8Encoder::encode) + // + // Set whole scaled buffer to black now so the empty + // space appears as "black bars". + rgba_scaled_buffer_.erase(); + } + + frame_->reset(pts, rgba_buffer_); + + return std::move(frame_); +} + +void VP8Encoder::encode(VideoFrame::instance_t frame) +{ + { + using T = YUV420pFrame; + + SRB2_ASSERT(frame_ == nullptr); + SRB2_ASSERT(dynamic_cast(frame.get()) != nullptr); + + frame_ = std::unique_ptr(static_cast(frame.release())); + } + + // This frame must be scaled to match encoder configuration + if (frame_->width() != width() || frame_->height() != height()) + { + rgba_scaled_buffer_.resize(width(), height()); + frame_->scale(rgba_scaled_buffer_); + } + else + { + rgba_scaled_buffer_.release(); + } + + frame_->convert(); + + if (vpx_codec_encode(ctx_, img_, frame_->pts(), 1, 0, deadline_) != VPX_CODEC_OK) + { + throw std::invalid_argument(fmt::format("VP8Encoder::encode: vpx_codec_encode: {}", VpxError(ctx_))); + } + + process(); +} + +void VP8Encoder::flush() +{ + do + { + if (vpx_codec_encode(ctx_, nullptr, 0, 0, 0, 0) != VPX_CODEC_OK) + { + throw std::invalid_argument(fmt::format("VP8Encoder::flush: vpx_codec_encode: {}", VpxError(ctx_))); + } + } while (process()); +} + +bool VP8Encoder::process() +{ + bool output = false; + + vpx_codec_iter_t iter = NULL; + const vpx_codec_cx_pkt_t* pkt; + + while ((pkt = vpx_codec_get_cx_data(ctx_, &iter))) + { + output = true; + + if (pkt->kind != VPX_CODEC_CX_FRAME_PKT) + { + continue; + } + + auto& frame = pkt->data.frame; + + { + const std::lock_guard _(frame_count_mutex_); + + duration_ = frame.pts + frame.duration; + frame_count_++; + } + + const float ts = frame.pts / static_cast(frame_rate()); + + using T = const std::byte; + tcb::span p(reinterpret_cast(frame.buf), frame.sz); + + write_frame(p, std::chrono::duration(ts), (frame.flags & VPX_FRAME_IS_KEY)); + } + + return output; +} + +template +void VP8Encoder::control(vp8e_enc_control_id id, const char* option) +{ + auto value = options_.get(option); + + if (vpx_codec_control_(ctx_, id, value) != VPX_CODEC_OK) + { + throw std::invalid_argument(fmt::format("vpx_codec_control: {}, {}={}", VpxError(ctx_), option, value)); + } +} + +VideoEncoder::FrameCount VP8Encoder::frame_count() const +{ + const std::lock_guard _(frame_count_mutex_); + + return {frame_count_, std::chrono::duration(duration_ / static_cast(frame_rate()))}; +} diff --git a/src/media/vp8.hpp b/src/media/vp8.hpp new file mode 100644 index 000000000..b4c68f24d --- /dev/null +++ b/src/media/vp8.hpp @@ -0,0 +1,102 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_VP8_HPP__ +#define __SRB2_MEDIA_VP8_HPP__ + +#include + +#include + +#include "options.hpp" +#include "video_encoder.hpp" +#include "yuv420p.hpp" + +namespace srb2::media +{ + +class VP8Encoder : public VideoEncoder +{ +public: + static const Options options_; + + VP8Encoder(VideoEncoder::Config config); + + virtual VideoFrame::instance_t new_frame(int width, int height, int pts) override final; + + virtual void encode(VideoFrame::instance_t frame) override final; + virtual void flush() override final; + + virtual const char* name() const override final { return "VP8"; } + virtual int width() const override final { return img_->w; } + virtual int height() const override final { return img_->h; } + virtual int frame_rate() const override final { return frame_rate_; } + virtual int thread_count() const override final { return thread_count_; } + + virtual FrameCount frame_count() const override final; + +private: + class CtxWrapper + { + public: + CtxWrapper(const Config config); + ~CtxWrapper(); + + operator vpx_codec_ctx_t*() { return &ctx_; } + operator vpx_codec_ctx_t&() { return ctx_; } + + private: + vpx_codec_ctx_t ctx_; + }; + + class ImgWrapper + { + public: + ImgWrapper(int width, int height); + ~ImgWrapper(); + + operator vpx_image_t*() { return &img_; } + vpx_image_t* operator->() { return &img_; } + const vpx_image_t* operator->() const { return &img_; } + + private: + vpx_image_t img_; + }; + + static vpx_codec_iface_t* kCodec; + + static const vpx_codec_enc_cfg_t configure(const Config config); + + CtxWrapper ctx_; + ImgWrapper img_; + + const int frame_rate_; + const int thread_count_ = options_.get("threads"); + const int deadline_ = options_.get("deadline"); + + mutable std::recursive_mutex frame_count_mutex_; + + int duration_ = 0; + int frame_count_ = 0; + + YUV420pFrame::BufferRGBA // + rgba_buffer_, + rgba_scaled_buffer_; // only allocated if input NEEDS scaling + + std::unique_ptr frame_; + + bool process(); + + template // T = option type + void control(vp8e_enc_control_id id, const char* option); +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_VP8_HPP__ diff --git a/src/media/vpx_error.hpp b/src/media/vpx_error.hpp new file mode 100644 index 000000000..06b502398 --- /dev/null +++ b/src/media/vpx_error.hpp @@ -0,0 +1,45 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_VPX_ERROR_HPP__ +#define __SRB2_MEDIA_VPX_ERROR_HPP__ + +#include + +#include +#include + +class VpxError +{ +public: + VpxError(vpx_codec_ctx_t& ctx) : ctx_(&ctx) {} + + std::string description() const + { + const char* error = vpx_codec_error(ctx_); + const char* detail = vpx_codec_error_detail(ctx_); + + return detail ? fmt::format("{}: {}", error, detail) : error; + } + +private: + vpx_codec_ctx_t* ctx_; +}; + +template <> +struct fmt::formatter : formatter +{ + template + auto format(const VpxError& error, FormatContext& ctx) const + { + return formatter::format(error.description(), ctx); + } +}; + +#endif // __SRB2_MEDIA_VPX_ERROR_HPP__ From 60899133c1711d6d4dca6c6766a7b75f3d95fa3e Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:12:55 -0800 Subject: [PATCH 06/18] media: add libwebm container --- src/media/CMakeLists.txt | 6 + src/media/cfile.cpp | 34 ++++++ src/media/cfile.hpp | 36 ++++++ src/media/webm.hpp | 26 ++++ src/media/webm_container.cpp | 228 +++++++++++++++++++++++++++++++++++ src/media/webm_container.hpp | 112 +++++++++++++++++ src/media/webm_writer.hpp | 32 +++++ 7 files changed, 474 insertions(+) create mode 100644 src/media/cfile.cpp create mode 100644 src/media/cfile.hpp create mode 100644 src/media/webm.hpp create mode 100644 src/media/webm_container.cpp create mode 100644 src/media/webm_container.hpp create mode 100644 src/media/webm_writer.hpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index 3e748c08a..5025a7550 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -1,5 +1,7 @@ target_sources(SRB2SDL2 PRIVATE audio_encoder.hpp + cfile.cpp + cfile.hpp container.hpp encoder.hpp options.cpp @@ -12,6 +14,10 @@ target_sources(SRB2SDL2 PRIVATE vp8.cpp vp8.hpp vpx_error.hpp + webm.hpp + webm_container.cpp + webm_container.hpp + webm_writer.hpp yuv420p.cpp yuv420p.hpp ) diff --git a/src/media/cfile.cpp b/src/media/cfile.cpp new file mode 100644 index 000000000..25c8ba019 --- /dev/null +++ b/src/media/cfile.cpp @@ -0,0 +1,34 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include +#include + +#include + +#include "cfile.hpp" + +using namespace srb2::media; + +CFile::CFile(const std::string file_name) : name_(file_name) +{ + file_ = std::fopen(name(), "wb"); + + if (file_ == nullptr) + { + throw std::invalid_argument(fmt::format("{}: {}", name(), std::strerror(errno))); + } +} + +CFile::~CFile() +{ + std::fclose(file_); +} diff --git a/src/media/cfile.hpp b/src/media/cfile.hpp new file mode 100644 index 000000000..7d0e8595e --- /dev/null +++ b/src/media/cfile.hpp @@ -0,0 +1,36 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_CFILE_HPP__ +#define __SRB2_MEDIA_CFILE_HPP__ + +#include +#include + +namespace srb2::media +{ + +class CFile +{ +public: + CFile(const std::string file_name); + ~CFile(); + + operator std::FILE*() const { return file_; } + + const char* name() const { return name_.c_str(); } + +private: + std::string name_; + std::FILE* file_; +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_CFILE_HPP__ diff --git a/src/media/webm.hpp b/src/media/webm.hpp new file mode 100644 index 000000000..2031f58f0 --- /dev/null +++ b/src/media/webm.hpp @@ -0,0 +1,26 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_WEBM_HPP__ +#define __SRB2_MEDIA_WEBM_HPP__ + +#include +#include +#include + +namespace srb2::media::webm +{ + +using track = uint64_t; +using timestamp = uint64_t; +using duration = std::chrono::duration; + +}; // namespace srb2::media::webm + +#endif // __SRB2_MEDIA_WEBM_HPP__ diff --git a/src/media/webm_container.cpp b/src/media/webm_container.cpp new file mode 100644 index 000000000..3d2ffc5b9 --- /dev/null +++ b/src/media/webm_container.cpp @@ -0,0 +1,228 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include + +#include "../cxxutil.hpp" +#include "webm_vorbis.hpp" +#include "webm_vp8.hpp" + +using namespace srb2::media; + +using time_unit_t = MediaEncoder::time_unit_t; + +WebmContainer::WebmContainer(const Config cfg) : writer_(cfg.file_name), dtor_cb_(cfg.destructor_callback) +{ + SRB2_ASSERT(segment_.Init(&writer_) == true); +} + +WebmContainer::~WebmContainer() +{ + flush_queue(); + + if (!segment_.Finalize()) + { + CONS_Alert(CONS_WARNING, "mkvmuxer::Segment::Finalize has failed\n"); + } + + finalized_ = true; + + if (dtor_cb_) + { + dtor_cb_(*this); + } +} + +std::unique_ptr WebmContainer::make_audio_encoder(AudioEncoder::Config cfg) +{ + const uint64_t tid = segment_.AddAudioTrack(cfg.sample_rate, cfg.channels, 0); + + return std::make_unique(*this, tid, cfg); +} + +std::unique_ptr WebmContainer::make_video_encoder(VideoEncoder::Config cfg) +{ + const uint64_t tid = segment_.AddVideoTrack(cfg.width, cfg.height, 0); + + return std::make_unique(*this, tid, cfg); +} + +time_unit_t WebmContainer::duration() const +{ + if (finalized_) + { + const auto& si = *segment_.segment_info(); + + return webm::duration(static_cast(si.duration() * si.timecode_scale())); + } + + auto _ = queue_guard(); + + return webm::duration(latest_timestamp_); +} + +std::size_t WebmContainer::size() const +{ + if (finalized_) + { + return writer_.Position(); + } + + auto _ = queue_guard(); + + return writer_.Position() + queue_size_; +} + +std::size_t WebmContainer::track_size(webm::track trackid) const +{ + auto _ = queue_guard(); + + return queue_.at(trackid).data_size; +} + +time_unit_t WebmContainer::track_duration(webm::track trackid) const +{ + auto _ = queue_guard(); + + return webm::duration(queue_.at(trackid).flushed_timestamp); +} + +void WebmContainer::write_frame( + tcb::span buffer, + webm::track trackid, + webm::timestamp timestamp, + bool is_key_frame +) +{ + SRB2_ASSERT( + segment_.AddFrame( + reinterpret_cast(buffer.data()), + buffer.size_bytes(), + trackid, + timestamp, + is_key_frame + ) == true + ); + + queue_[trackid].data_size += buffer.size_bytes(); +} + +void WebmContainer::queue_frame( + tcb::span buffer, + webm::track trackid, + webm::timestamp timestamp, + bool is_key_frame +) +{ + auto _ = queue_guard(); + + auto& q = queue_.at(trackid); + + // If another track is behind this one, queue this + // frame until the other track catches up. + + if (flush_queue() < timestamp) + { + q.frames.emplace_back(buffer, timestamp, is_key_frame); + queue_size_ += buffer.size_bytes(); + } + else + { + // Nothing is waiting; this frame can be written + // immediately. + + write_frame(buffer, trackid, timestamp, is_key_frame); + q.flushed_timestamp = timestamp; + } + + q.queued_timestamp = timestamp; + latest_timestamp_ = timestamp; +} + +webm::timestamp WebmContainer::flush_queue() +{ + webm::timestamp goal = latest_timestamp_; + + // Flush all tracks' queues, not beyond the end of the + // shortest track. + + for (const auto& [_, q] : queue_) + { + if (q.queued_timestamp < goal) + { + goal = q.queued_timestamp; + } + } + + webm::timestamp shortest; + + do + { + shortest = goal; + + for (const auto& [tid, q] : queue_) + { + const webm::timestamp flushed = flush_single_queue(tid, q.queued_timestamp); + + if (flushed < shortest) + { + shortest = flushed; + } + } + } while (shortest < goal); + + return shortest; +} + +webm::timestamp WebmContainer::flush_single_queue(webm::track trackid, webm::timestamp flushed_timestamp) +{ + webm::timestamp goal = flushed_timestamp; + + // Find the lowest timestamp yet flushed from all other + // tracks. We cannot write a frame beyond this timestamp + // because PTS must only increase. + + for (const auto& [tid, other] : queue_) + { + if (tid != trackid && other.flushed_timestamp < goal) + { + goal = other.flushed_timestamp; + } + } + + auto& q = queue_.at(trackid); + auto it = q.frames.cbegin(); + + // Flush previously queued frames in this track. + + for (; it != q.frames.cend(); ++it) + { + const auto& frame = *it; + + if (frame.timestamp > goal) + { + q.flushed_timestamp = frame.timestamp; + break; + } + + write_frame(frame.buffer, trackid, frame.timestamp, frame.is_key_frame); + + queue_size_ -= frame.buffer.size(); + } + + q.frames.erase(q.frames.cbegin(), it); + + if (q.frames.empty()) + { + q.flushed_timestamp = flushed_timestamp; + } + + return goal; +} diff --git a/src/media/webm_container.hpp b/src/media/webm_container.hpp new file mode 100644 index 000000000..e763515c8 --- /dev/null +++ b/src/media/webm_container.hpp @@ -0,0 +1,112 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_WEBM_CONTAINER_HPP__ +#define __SRB2_MEDIA_WEBM_CONTAINER_HPP__ + +#include +#include +#include +#include + +#include + +#include "container.hpp" +#include "webm.hpp" +#include "webm_writer.hpp" + +namespace srb2::media +{ + +class WebmContainer : virtual public MediaContainer +{ +public: + WebmContainer(Config cfg); + ~WebmContainer(); + + virtual std::unique_ptr make_audio_encoder(AudioEncoder::Config config) override final; + virtual std::unique_ptr make_video_encoder(VideoEncoder::Config config) override final; + + virtual const char* name() const override final { return "WebM"; } + virtual const char* file_name() const override final { return writer_.name(); } + + virtual time_unit_t duration() const override final; + virtual std::size_t size() const override final; + + std::size_t track_size(webm::track trackid) const; + time_unit_t track_duration(webm::track trackid) const; + + template + T* get_track(webm::track trackid) const + { + return reinterpret_cast(segment_.GetTrackByNumber(trackid)); + } + + void init_queue(webm::track trackid) { queue_.try_emplace(trackid); } + + // init_queue MUST be called before using this function. + void queue_frame( + tcb::span buffer, + webm::track trackid, + webm::timestamp timestamp, + bool is_key_frame + ); + + auto queue_guard() const { return std::lock_guard(queue_mutex_); } + +private: + struct FrameQueue + { + struct Frame + { + std::vector buffer; + webm::timestamp timestamp; + bool is_key_frame; + + Frame(tcb::span buffer_, webm::timestamp timestamp_, bool is_key_frame_) : + buffer(buffer_.begin(), buffer_.end()), timestamp(timestamp_), is_key_frame(is_key_frame_) + { + } + }; + + std::vector frames; + std::size_t data_size = 0; + + webm::timestamp flushed_timestamp = 0; + webm::timestamp queued_timestamp = 0; + }; + + mkvmuxer::Segment segment_; + WebmWriter writer_; + + mutable std::recursive_mutex queue_mutex_; + + std::unordered_map queue_; + + webm::timestamp latest_timestamp_ = 0; + std::size_t queue_size_ = 0; + + bool finalized_ = false; + const dtor_cb_t dtor_cb_; + + void write_frame( + tcb::span buffer, + webm::track trackid, + webm::timestamp timestamp, + bool is_key_frame + ); + + // Returns the largest timestamp that can be written. + webm::timestamp flush_queue(); + webm::timestamp flush_single_queue(webm::track trackid, webm::timestamp flushed_timestamp); +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_WEBM_CONTAINER_HPP__ diff --git a/src/media/webm_writer.hpp b/src/media/webm_writer.hpp new file mode 100644 index 000000000..50e091baa --- /dev/null +++ b/src/media/webm_writer.hpp @@ -0,0 +1,32 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_WEBM_WRITER_HPP__ +#define __SRB2_MEDIA_WEBM_WRITER_HPP__ + +#include +#include + +#include + +#include "cfile.hpp" + +namespace srb2::media +{ + +class WebmWriter : public CFile, public mkvmuxer::MkvWriter +{ +public: + WebmWriter(const std::string file_name) : CFile(file_name), MkvWriter(static_cast(*this)) {} + ~WebmWriter() { MkvWriter::Close(); } +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_WEBM_WRITER_HPP__ From 654f97fa72e2e301ab32f2626ea8ca26c540a278 Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:15:23 -0800 Subject: [PATCH 07/18] media: add WebM Vorbis and VP8 encoders --- src/media/CMakeLists.txt | 4 ++ src/media/webm_encoder.hpp | 51 ++++++++++++++++++++++ src/media/webm_vorbis.hpp | 59 +++++++++++++++++++++++++ src/media/webm_vorbis_lace.cpp | 79 ++++++++++++++++++++++++++++++++++ src/media/webm_vp8.hpp | 44 +++++++++++++++++++ 5 files changed, 237 insertions(+) create mode 100644 src/media/webm_encoder.hpp create mode 100644 src/media/webm_vorbis.hpp create mode 100644 src/media/webm_vorbis_lace.cpp create mode 100644 src/media/webm_vp8.hpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index 5025a7550..8282d847f 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -15,8 +15,12 @@ target_sources(SRB2SDL2 PRIVATE vp8.hpp vpx_error.hpp webm.hpp + webm_encoder.hpp webm_container.cpp webm_container.hpp + webm_vorbis.hpp + webm_vorbis_lace.cpp + webm_vp8.hpp webm_writer.hpp yuv420p.cpp yuv420p.hpp diff --git a/src/media/webm_encoder.hpp b/src/media/webm_encoder.hpp new file mode 100644 index 000000000..197886dba --- /dev/null +++ b/src/media/webm_encoder.hpp @@ -0,0 +1,51 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_WEBM_ENCODER_HPP__ +#define __SRB2_MEDIA_WEBM_ENCODER_HPP__ + +#include + +#include "encoder.hpp" +#include "webm_container.hpp" + +namespace srb2::media +{ + +template +class WebmEncoder : virtual public MediaEncoder +{ +public: + WebmEncoder(WebmContainer& container, webm::track trackid) : container_(container), trackid_(trackid) + { + container_.init_queue(trackid_); + } + +protected: + WebmContainer& container_; + webm::track trackid_; + + std::size_t size() const { return container_.track_size(trackid_); } + time_unit_t duration() const { return container_.track_duration(trackid_); } + + static T* get_track(const WebmContainer& container, webm::track trackid) { return container.get_track(trackid); } + + T* track() const { return get_track(container_, trackid_); } + + virtual void write_frame(frame_buffer_t p, time_unit_t ts, bool is_key_frame) override final + { + const auto ts_nano = std::chrono::duration_cast(ts); + + container_.queue_frame(p, trackid_, ts_nano.count(), is_key_frame); + } +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_WEBM_ENCODER_HPP__ diff --git a/src/media/webm_vorbis.hpp b/src/media/webm_vorbis.hpp new file mode 100644 index 000000000..5e3825ceb --- /dev/null +++ b/src/media/webm_vorbis.hpp @@ -0,0 +1,59 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_WEBM_VORBIS_HPP__ +#define __SRB2_MEDIA_WEBM_VORBIS_HPP__ + +#include +#include +#include +#include + +#include "../cxxutil.hpp" +#include "vorbis.hpp" +#include "webm_encoder.hpp" + +namespace srb2::media +{ + +class WebmVorbisEncoder : public WebmEncoder, public VorbisEncoder +{ +public: + WebmVorbisEncoder(WebmContainer& container, webm::track trackid, AudioEncoder::Config cfg) : + WebmEncoder(container, trackid), VorbisEncoder(cfg) + { + // write Vorbis extra data + + const auto p = make_vorbis_private_data(); + + SRB2_ASSERT(track()->SetCodecPrivate(reinterpret_cast(p.data()), p.size()) == true); + } + + virtual BitRate estimated_bit_rate() const override final + { + auto _ = container_.queue_guard(); + + const std::chrono::duration t = duration(); + + if (t <= t.zero()) + { + return {}; + } + + using namespace std::chrono_literals; + return {static_cast((size() * 8) / t.count()), 1s}; + } + +private: + std::vector make_vorbis_private_data(); +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_WEBM_VORBIS_HPP__ diff --git a/src/media/webm_vorbis_lace.cpp b/src/media/webm_vorbis_lace.cpp new file mode 100644 index 000000000..506d3f763 --- /dev/null +++ b/src/media/webm_vorbis_lace.cpp @@ -0,0 +1,79 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include +#include + +#include + +#include "webm_vorbis.hpp" + +// https://www.matroska.org/technical/notes.html#xiph-lacing +// https://www.matroska.org/technical/codec_specs.html#a_vorbis + +using namespace srb2::media; + +static std::size_t lace_length(const ogg_packet& op) +{ + return (op.bytes / 255) + 1; +} + +static void lace(std::vector& v, const ogg_packet& op) +{ + // The lacing size is encoded in at least one byte. If + // the value is 255, add the value of the next byte in + // sequence. This ends with a byte that is less than 255. + + std::fill_n(std::back_inserter(v), lace_length(op) - 1, std::byte {255}); + + const unsigned char n = (op.bytes % 255); + v.emplace_back(std::byte {n}); +} + +std::vector WebmVorbisEncoder::make_vorbis_private_data() +{ + const headers_t packets = generate_headers(); + + std::vector v; + + // There are three Vorbis header packets. The lacing for + // these packets in Matroska does not count the final + // packet. + + // clang-format off + v.reserve( + 1 + + lace_length(packets[0]) + + lace_length(packets[1]) + + packets[0].bytes + + packets[1].bytes + + packets[2].bytes); + // clang-format on + + // The first byte is the number of packets. Once again, + // the last packet is not counted. + v.emplace_back(std::byte {2}); + + // Then the laced sizes for each packet. + lace(v, packets[0]); + lace(v, packets[1]); + + // Then each packet's data. The last packet's data + // actually is written here. + for (auto op : packets) + { + tcb::span p(reinterpret_cast(op.packet), op.bytes); + + std::copy(p.begin(), p.end(), std::back_inserter(v)); + } + + return v; +} diff --git a/src/media/webm_vp8.hpp b/src/media/webm_vp8.hpp new file mode 100644 index 000000000..44e951146 --- /dev/null +++ b/src/media/webm_vp8.hpp @@ -0,0 +1,44 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_WEBM_VP8_HPP__ +#define __SRB2_MEDIA_WEBM_VP8_HPP__ + +#include "vp8.hpp" +#include "webm_encoder.hpp" + +namespace srb2::media +{ + +class WebmVP8Encoder : public WebmEncoder, public VP8Encoder +{ +public: + WebmVP8Encoder(WebmContainer& container, webm::track trackid, VideoEncoder::Config cfg) : + WebmEncoder(container, trackid), VP8Encoder(cfg) + { + } + + virtual BitRate estimated_bit_rate() const override final + { + auto _ = container_.queue_guard(); + + const int frames = frame_count().frames; + + if (frames <= 0) + { + return {}; + } + + return {(size() * 8) / frames, std::chrono::duration(1.f / frame_rate())}; + } +}; + +}; // namespace srb2::media + +#endif // __SRB2_MEDIA_WEBM_VP8_HPP__ From 82251f6fb60653fbe6365ec2686ff31846cc1fc8 Mon Sep 17 00:00:00 2001 From: James R Date: Sun, 12 Feb 2023 02:28:30 -0800 Subject: [PATCH 08/18] media: add core AVRecorder Generic interface to audio and video encoders. --- src/media/CMakeLists.txt | 4 + src/media/avrecorder.cpp | 211 +++++++++++++++++++++++++++++++++ src/media/avrecorder.hpp | 76 ++++++++++++ src/media/avrecorder_impl.hpp | 164 +++++++++++++++++++++++++ src/media/avrecorder_queue.cpp | 142 ++++++++++++++++++++++ 5 files changed, 597 insertions(+) create mode 100644 src/media/avrecorder.cpp create mode 100644 src/media/avrecorder.hpp create mode 100644 src/media/avrecorder_impl.hpp create mode 100644 src/media/avrecorder_queue.cpp diff --git a/src/media/CMakeLists.txt b/src/media/CMakeLists.txt index 8282d847f..ec387a428 100644 --- a/src/media/CMakeLists.txt +++ b/src/media/CMakeLists.txt @@ -1,5 +1,9 @@ target_sources(SRB2SDL2 PRIVATE audio_encoder.hpp + avrecorder.cpp + avrecorder.hpp + avrecorder_impl.hpp + avrecorder_queue.cpp cfile.cpp cfile.hpp container.hpp diff --git a/src/media/avrecorder.cpp b/src/media/avrecorder.cpp new file mode 100644 index 000000000..ef1962fed --- /dev/null +++ b/src/media/avrecorder.cpp @@ -0,0 +1,211 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cxxutil.hpp" +#include "../i_time.h" +#include "../m_fixed.h" +#include "avrecorder_impl.hpp" +#include "webm_container.hpp" + +using namespace srb2::media; + +using Impl = AVRecorder::Impl; + +namespace +{ + +constexpr auto kBufferMethod = VideoFrame::BufferMethod::kEncoderAllocatedRGBA8888; + +}; // namespace + +Impl::Impl(Config cfg) : + max_size_(cfg.max_size), + max_duration_(cfg.max_duration), + + container_(std::make_unique(MediaContainer::Config { + .file_name = cfg.file_name, + })), + + audio_encoder_(make_audio_encoder(cfg)), + video_encoder_(make_video_encoder(cfg)), + + epoch_(I_GetTime()), + + thread_([this] { worker(); }) +{ +} + +std::unique_ptr Impl::make_audio_encoder(const Config cfg) const +{ + if (!cfg.audio) + { + return nullptr; + } + + const Config::Audio& a = *cfg.audio; + + return container_->make_audio_encoder({ + .channels = 2, + .sample_rate = a.sample_rate, + }); +} + +std::unique_ptr Impl::make_video_encoder(const Config cfg) const +{ + if (!cfg.video) + { + return nullptr; + } + + const Config::Video& v = *cfg.video; + + return container_->make_video_encoder({ + .width = v.width, + .height = v.height, + .frame_rate = v.frame_rate, + .buffer_method = kBufferMethod, + }); +} + +Impl::~Impl() +{ + valid_ = false; + wake_up_worker(); + thread_.join(); + + try + { + // Finally flush encoders, unless queues were finished + // already due to time or size constraints. + + if (!audio_queue_.finished()) + { + audio_encoder_->flush(); + } + + if (!video_queue_.finished()) + { + video_encoder_->flush(); + } + } + catch (const std::exception& ex) + { + CONS_Alert(CONS_ERROR, "AVRecorder::Impl::~Impl: %s\n", ex.what()); + return; + } +} + +std::optional Impl::advance_video_pts() +{ + auto _ = queue_guard(); + + // Don't let this queue grow out of hand. It's normal + // for encoding time to vary by a small margin and + // spend longer than one frame rate on a single + // frame. It should normalize though. + + if (video_queue_.vec_.size() >= 3) + { + return {}; + } + + SRB2_ASSERT(video_encoder_ != nullptr); + + const float tic_pts = video_encoder_->frame_rate() / static_cast(TICRATE); + const int pts = ((I_GetTime() - epoch_) + FixedToFloat(g_time.timefrac)) * tic_pts; + + if (!video_queue_.advance(pts, 1)) + { + return {}; + } + + return pts; +} + +void Impl::worker() +{ + for (;;) + { + QueueState qs; + + try + { + while ((qs = encode_queues()) == QueueState::kFlushed) + ; + } + catch (const std::exception& ex) + { + CONS_Alert(CONS_ERROR, "AVRecorder::Impl::worker: %s\n", ex.what()); + break; + } + + if (qs != QueueState::kFinished && valid_) + { + std::unique_lock lock(queue_mutex_); + + queue_cond_.wait(lock); + } + else + { + break; + } + } + + // Breaking out of the loop ensures invalidation! + valid_ = false; +} + +AVRecorder::AVRecorder(const Config config) : impl_(std::make_unique(config)) +{ +} + +AVRecorder::~AVRecorder() +{ + // impl_ is destroyed in a background thread so it doesn't + // block the thread AVRecorder was destroyed in. + // + // TODO: Save into a thread pool instead of detaching so + // the thread could be joined at program exit and + // not possibly terminate before fully destroyed? + + std::thread([_ = std::move(impl_)] {}).detach(); +} + +void AVRecorder::push_audio_samples(audio_buffer_t buffer) +{ + const auto _ = impl_->queue_guard(); + + auto& q = impl_->audio_queue_; + + if (!q.advance(q.pts(), buffer.size())) + { + return; + } + + using T = const float; + tcb::span p(reinterpret_cast(buffer.data()), buffer.size() * 2); // 2 channels + + std::copy(p.begin(), p.end(), std::back_inserter(q.vec_)); + + impl_->wake_up_worker(); +} + +bool AVRecorder::invalid() const +{ + return !impl_->valid_; +} diff --git a/src/media/avrecorder.hpp b/src/media/avrecorder.hpp new file mode 100644 index 000000000..45834ec08 --- /dev/null +++ b/src/media/avrecorder.hpp @@ -0,0 +1,76 @@ +// RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by James Robert Roman +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef __SRB2_MEDIA_AVRECORDER_HPP__ +#define __SRB2_MEDIA_AVRECORDER_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../audio/sample.hpp" + +namespace srb2::media +{ + +class AVRecorder +{ +public: + using audio_sample_t = srb2::audio::Sample<2>; + using audio_buffer_t = tcb::span; + + class Impl; + + struct Config + { + struct Audio + { + int sample_rate; + }; + + struct Video + { + int width; + int height; + int frame_rate; + }; + + std::string file_name; + + std::optional max_size; // file size limit + std::optional> max_duration; + + std::optional