diff --git a/CMakeLists.txt b/CMakeLists.txt index bed7e71..da2af6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,7 @@ find_package(Protobuf REQUIRED) find_package(OpenSSL REQUIRED) find_package(rtaudio REQUIRED) find_package(aasdk REQUIRED) +find_package(h264 REQUIRED) find_package(Threads) include(${base_directory}/cmake_modules/gitversion.cmake) diff --git a/cmake_modules/Findh264.cmake b/cmake_modules/Findh264.cmake new file mode 100644 index 0000000..0f9bffd --- /dev/null +++ b/cmake_modules/Findh264.cmake @@ -0,0 +1,26 @@ +set (H264_INCLUDE_DIR /usr/local/include/h264bitstream) +set (H264_LIB_DIR /usr/local/lib) + +set(H264_FOUND TRUE) + +if (H264_FOUND) + if (NOT h264_FIND_QUIETLY) + message(STATUS "Found h264bitstream:") + message(STATUS " - Includes: ${H264_INCLUDE_DIR}") + message(STATUS " - Libraries: ${H264_LIB_DIR}") + endif() + add_library(h264 INTERFACE) + target_include_directories(h264 INTERFACE ${H264_INCLUDE_DIR}) + set_target_properties(h264 PROPERTIES INTERFACE_LINK_DIRECTORIES ${H264_LIB_DIR}) + target_link_libraries(h264 INTERFACE libh264bitstream.so) +else() + if (h264_FIND_REQUIRED) + if(H264_INCLUDE_DIR) + message(FATAL_ERROR "h264 was found but not built. Perform an in-source build.") + else() + message(FATAL_ERROR "Could not find h264") + endif() + endif() +endif() + +mark_as_advanced(H264_INCLUDE_DIR H264_LIBRARIES) diff --git a/include/openauto/Configuration/Configuration.hpp b/include/openauto/Configuration/Configuration.hpp index 6f92f73..e22b71e 100644 --- a/include/openauto/Configuration/Configuration.hpp +++ b/include/openauto/Configuration/Configuration.hpp @@ -50,6 +50,8 @@ public: int32_t getOMXLayerIndex() const override; void setVideoMargins(QRect value) override; QRect getVideoMargins() const override; + void setWhitescreenWorkaround(bool value) override; + bool getWhitescreenWorkaround() const override; bool getTouchscreenEnabled() const override; void setTouchscreenEnabled(bool value) override; @@ -91,6 +93,7 @@ private: size_t screenDPI_; int32_t omxLayerIndex_; QRect videoMargins_; + bool whitescreenWorkaround_; bool enableTouchscreen_; ButtonCodes buttonCodes_; BluetoothAdapterType bluetoothAdapterType_; @@ -115,6 +118,7 @@ private: static const std::string cVideoOMXLayerIndexKey; static const std::string cVideoMarginWidth; static const std::string cVideoMarginHeight; + static const std::string cVideoWhitescreenWorkaround; static const std::string cAudioMusicAudioChannelEnabled; static const std::string cAudioSpeechAudioChannelEnabled; diff --git a/include/openauto/Configuration/IConfiguration.hpp b/include/openauto/Configuration/IConfiguration.hpp index 68d9b0b..2d3edd8 100644 --- a/include/openauto/Configuration/IConfiguration.hpp +++ b/include/openauto/Configuration/IConfiguration.hpp @@ -59,6 +59,8 @@ public: virtual int32_t getOMXLayerIndex() const = 0; virtual void setVideoMargins(QRect value) = 0; virtual QRect getVideoMargins() const = 0; + virtual void setWhitescreenWorkaround(bool value) = 0; + virtual bool getWhitescreenWorkaround() const = 0; virtual bool getTouchscreenEnabled() const = 0; virtual void setTouchscreenEnabled(bool value) = 0; diff --git a/include/openauto/Projection/GSTVideoOutput.hpp b/include/openauto/Projection/GSTVideoOutput.hpp index 3b6df7c..646df14 100644 --- a/include/openauto/Projection/GSTVideoOutput.hpp +++ b/include/openauto/Projection/GSTVideoOutput.hpp @@ -78,6 +78,8 @@ private: static GstPadProbeReturn convertProbe(GstPad* pad, GstPadProbeInfo* info, void*); static gboolean busCallback(GstBus*, GstMessage* message, gpointer*); + bool firstHeaderParsed = false; + QGst::ElementPtr videoSink_; QQuickWidget* videoWidget_; GstElement* vidPipeline_; diff --git a/openauto/CMakeLists.txt b/openauto/CMakeLists.txt index e201c6c..8f2238a 100644 --- a/openauto/CMakeLists.txt +++ b/openauto/CMakeLists.txt @@ -115,6 +115,7 @@ target_link_libraries(openauto PUBLIC Threads::Threads ${Boost_LIBRARIES} aasdk + h264 rtaudio Qt5::Bluetooth Qt5::MultimediaWidgets diff --git a/openauto/Configuration/Configuration.cpp b/openauto/Configuration/Configuration.cpp index bf53f44..900ec58 100644 --- a/openauto/Configuration/Configuration.cpp +++ b/openauto/Configuration/Configuration.cpp @@ -35,6 +35,8 @@ const std::string Configuration::cVideoScreenDPIKey = "Video.ScreenDPI"; const std::string Configuration::cVideoOMXLayerIndexKey = "Video.OMXLayerIndex"; const std::string Configuration::cVideoMarginWidth = "Video.MarginWidth"; const std::string Configuration::cVideoMarginHeight = "Video.MarginHeight"; +const std::string Configuration::cVideoWhitescreenWorkaround = "Video.WhitesreenWorkaround"; + const std::string Configuration::cAudioMusicAudioChannelEnabled = "Audio.MusicAudioChannelEnabled"; const std::string Configuration::cAudioSpeechAudioChannelEnabled = "Audio.SpeechAudioChannelEnabled"; @@ -93,6 +95,7 @@ void Configuration::load() omxLayerIndex_ = iniConfig.get(cVideoOMXLayerIndexKey, 1); videoMargins_ = QRect(0, 0, iniConfig.get(cVideoMarginWidth, 0), iniConfig.get(cVideoMarginHeight, 0)); + whitescreenWorkaround_ = iniConfig.get(cVideoWhitescreenWorkaround, true); enableTouchscreen_ = iniConfig.get(cInputEnableTouchscreenKey, true); this->readButtonCodes(iniConfig); @@ -130,6 +133,7 @@ void Configuration::reset() screenDPI_ = 140; omxLayerIndex_ = 1; videoMargins_ = QRect(0, 0, 0, 0); + whitescreenWorkaround_ = true; enableTouchscreen_ = true; buttonCodes_.clear(); bluetoothAdapterType_ = BluetoothAdapterType::NONE; @@ -151,6 +155,7 @@ void Configuration::save() iniConfig.put(cVideoOMXLayerIndexKey, omxLayerIndex_); iniConfig.put(cVideoMarginWidth, videoMargins_.width()); iniConfig.put(cVideoMarginHeight, videoMargins_.height()); + iniConfig.put(cVideoWhitescreenWorkaround, whitescreenWorkaround_); iniConfig.put(cInputEnableTouchscreenKey, enableTouchscreen_); this->writeButtonCodes(iniConfig); @@ -240,6 +245,16 @@ QRect Configuration::getVideoMargins() const return videoMargins_; } +void Configuration::setWhitescreenWorkaround(bool value) +{ + whitescreenWorkaround_ = value; +} + +bool Configuration::getWhitescreenWorkaround() const +{ + return whitescreenWorkaround_; +} + bool Configuration::getTouchscreenEnabled() const { return enableTouchscreen_; diff --git a/openauto/Projection/GSTVideoOutput.cpp b/openauto/Projection/GSTVideoOutput.cpp index 4513d32..f76a9d5 100644 --- a/openauto/Projection/GSTVideoOutput.cpp +++ b/openauto/Projection/GSTVideoOutput.cpp @@ -20,7 +20,13 @@ #include "aasdk/Common/Data.hpp" #include "openauto/Projection/GSTVideoOutput.hpp" #include "OpenautoLog.hpp" +#include "h264_stream.h" #include +// these are needed only for pretty printing of data, to be removed +#include +#include +#include +#include namespace openauto { @@ -176,17 +182,103 @@ bool GSTVideoOutput::init() void GSTVideoOutput::write(uint64_t timestamp, const aasdk::common::DataConstBuffer& buffer) { - GstBuffer* buffer_ = gst_buffer_new_and_alloc(buffer.size); - gst_buffer_fill(buffer_, 0, buffer.cdata, buffer.size); - int ret = gst_app_src_push_buffer((GstAppSrc*)vidSrc_, buffer_); - if(ret != GST_FLOW_OK) + if(!firstHeaderParsed && this->configuration_->getWhitescreenWorkaround()) { - OPENAUTO_LOG(info) << "[GSTVideoOutput] push buffer returned " << ret << " for " << buffer.size << "bytes"; + // I really really really hate this. + + // Raspberry Pi hardware h264 decode appears broken if video_signal_type VUI parameters are given in the h264 header + // And we don't have control over Android Auto putting these parameters in (which it does.. on some model phones) + // So we pull in h264bitstream to edit this header on the fly + + // This is not a fix, I want to be very clear about that. I don't know what else I'm breaking, or run the + // risk of breaking by doing this. This code should only remain here as long as the Pi Engineers haven't released + // a firmware/driver fix for this yet. An issue has been opened upstream at https://github.com/raspberrypi/firmware/issues/1673 + + // Android Auto seems nice enough to always start a message with a new h264 packet, + // but that doesn't mean we don't have multiple within the message. + // So if we have a message that _could_ fit two packets (which are delimited by 0x00000001) + // then we try to find the second and save the data it contains, while editing and replacing the first. + + // This header should also always be within the first video message we receive from a device... I think + + std::vector delimit_sequence{0x00, 0x00, 0x00, 0x01}; + std::vector::iterator sequence_split; + std::vector incoming_buffer(&buffer.cdata[0], &buffer.cdata[buffer.size]); + + int nal_start, nal_end; + uint8_t* buf = (uint8_t *) buffer.cdata; + int len = buffer.size; + h264_stream_t* h = h264_new(); + // finds the first NAL packet + find_nal_unit(buf, len, &nal_start, &nal_end); + // parses it + read_nal_unit(h, &buf[nal_start], nal_end - nal_start); + // wipe all the color description stuff that breaks Pis + h->sps->vui.video_signal_type_present_flag = 0x00; + h->sps->vui.video_format = 0x00; + h->sps->vui.video_full_range_flag = 0x00; + h->sps->vui.colour_description_present_flag = 0x00; + h->sps->vui.video_format = 0x00; + h->sps->vui.video_full_range_flag = 0x00; + h->sps->vui.colour_description_present_flag = 0x00; + h->sps->vui.colour_primaries = 0x00; + h->sps->vui.transfer_characteristics = 0x00; + h->sps->vui.matrix_coefficients = 0x00; + + // grab some storage + uint8_t* out_buf = new uint8_t[30]; + + // write it to the storage's 3rd index, because h264bitstream seems to have a bug + // where it both doesn't write the delimiter, and it prepends a leading 0x00 + len = write_nal_unit(h, &out_buf[3], 30) + 3; + // write the delimiter back + out_buf[0] = 0x00; + out_buf[1] = 0x00; + out_buf[2] = 0x00; + out_buf[3] = 0x01; + + // output to gstreamer + GstBuffer* buffer_ = gst_buffer_new_and_alloc(len); + gst_buffer_fill(buffer_, 0, out_buf, len); + int ret = gst_app_src_push_buffer((GstAppSrc*)vidSrc_, buffer_); + if(ret != GST_FLOW_OK) + { + OPENAUTO_LOG(info) << "[GSTVideoOutput] Injecting header failed"; + } + + // then check if there's data we need to save + if(incoming_buffer.size() >= 8){ + sequence_split = std::search(incoming_buffer.begin()+4, incoming_buffer.end(), delimit_sequence.begin(), delimit_sequence.end()); + if(sequence_split != incoming_buffer.end()){ + std::vector incoming_data_saved(sequence_split, incoming_buffer.end()); + GstBuffer* buffer_ = gst_buffer_new_and_alloc(incoming_data_saved.size()); + gst_buffer_fill(buffer_, 0, incoming_data_saved.data(), incoming_data_saved.size()); + int ret = gst_app_src_push_buffer((GstAppSrc*)vidSrc_, buffer_); + if(ret != GST_FLOW_OK) + { + OPENAUTO_LOG(info) << "[GSTVideoOutput] Injecting partial header failed"; + } + } + } + OPENAUTO_LOG(info) << "[GSTVideoOutput] Intercepted and replaced h264 header"; + + firstHeaderParsed=true; + } + else + { + GstBuffer* buffer_ = gst_buffer_new_and_alloc(buffer.size); + gst_buffer_fill(buffer_, 0, buffer.cdata, buffer.size); + int ret = gst_app_src_push_buffer((GstAppSrc*)vidSrc_, buffer_); + if(ret != GST_FLOW_OK) + { + OPENAUTO_LOG(info) << "[GSTVideoOutput] push buffer returned " << ret << " for " << buffer.size << "bytes"; + } } } void GSTVideoOutput::onStartPlayback() { + firstHeaderParsed = false; if(activeCallback_ != nullptr) { activeCallback_(true); @@ -215,6 +307,8 @@ void GSTVideoOutput::stop() void GSTVideoOutput::onStopPlayback() { + firstHeaderParsed = false; + if(activeCallback_ != nullptr) { activeCallback_(false);