video.h

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
#pragma once
 
#include <condition_variable>
#include <queue>
#include <thread>
#include <mutex>
#include <climits>
#include <cassert>
#include <nap/resource.h>
#include <rtti/factory.h>
#include <utility/autoresetevent.h>
#include <nap/signalslot.h>
 
struct AVPacket;
struct AVCodec;
struct AVCodecParserContext;
struct AVCodecContext;
struct AVFormatContext;
struct AVFrame;
struct SwrContext;
struct AVDictionary;
struct AVStream;
 
namespace nap
{
    // Forward Declares
    class Video;
    namespace audio
    {
        class VideoNode;
    }
 
    class AudioFormat
    {
    public:
        enum class EChannelLayout : uint8_t
        {
            Mono,
            Stereo,
            _2Point1,
            _2_1,
            Surround,
            _3Point1,
            _4Point0,
            _4Point1,
            _2_2,
            Quad,
            _5Point0,
            _5Point1,
            _5Point0_Back,
            _5Point1_Back,
            _6Point0,
            _6Point0_Front,
            Hexagonal,
            _6Point1,
            _6Point1_Back,
            _6Point1_Front,
            _7Point0,
            _7Point0_Front,
            _7Point1,
            _7Point1_Wide,
            _7Point1_Wide_Back,
            Octagonal,
            Hexadecagonal,
            Stereo_Downmix
        };
 
        enum class ESampleFormat 
        {
            U8,         
            S16,        
            S32,        
            FLT,        
            DBL,        
            S64         
        };
 
        AudioFormat(EChannelLayout channelLayout, ESampleFormat sampleFormat, int sampleRate);
 
        AudioFormat(int numChannels, ESampleFormat sampleFormat, int sampleRate);
 
        int getSampleRate() const { return mSampleRate; }
 
        int64_t getChannelLayout() const { return mChannelLayout; }
 
        int getSampleFormat() const { return mSampleFormat; }
 
    private:
        int         mSampleRate;            
        int64_t     mChannelLayout;         
        int         mSampleFormat;          
    };
 
 
    struct NAPAPI Frame
    {
        bool isValid() const { return mFrame != nullptr; }
        void free();
 
        AVFrame*    mFrame = nullptr;       
        double      mPTSSecs = 0.0;         
        int         mFirstPacketDTS = 0;    
    };
 
 
    class NAPAPI AVState final
    {
    public:
        using OnClearFrameQueueFunction = std::function<void()>;
 
        AVState(Video& video);
 
        ~AVState();
         
        void init(int stream, AVCodec* codec, AVCodecContext* codecContext);
 
        void close();
 
        bool isValid() const { return mStream != -1; }
 
        int getStream() const { return mStream; }
 
        void startDecodeThread(const OnClearFrameQueueFunction& onClearFrameQueueFunction = OnClearFrameQueueFunction());
 
        void exitDecodeThread(bool join);
 
        bool isFinished() const;
 
        void clearFrameQueue();
 
        void clearPacketQueue();
 
        bool matchesStream(const AVPacket& packet) const;
 
        bool addSeekStartPacket();
 
        bool addSeekEndPacket(double seekTargetSecs);
 
        bool addEndOfFilePacket();
 
        bool addIOFinishedPacket();
 
        bool addPacket(AVPacket& packet);
 
        bool waitForFrameQueueEmpty(bool& exitIOThreadSignalled);
 
        void cancelWaitForFrameQueueEmpty();
 
        void resetWaitForFrameQueueEmpty();
 
        bool waitForEndOfFileProcessed();
 
        void cancelWaitForEndOfFileProcessed();
 
        void resetEndOfFileProcessed();
 
        void waitSeekStartPacketProcessed();
        
        int waitForReceiveFrame();
 
        void drainSeekFrameQueue();
 
        Frame popFrame();
 
        Frame tryPopFrame(double pts);
 
        Frame peekFrame();
 
        Frame popSeekFrame();
 
        void notifyStartIOThread();
 
        void notifyExitIOThread();
 
        AVCodec& getCodec() { return *mCodec; }
 
        AVCodecContext& getCodecContext() { return *mCodecContext; }
 
    private:
        enum class EDecodeFrameResult
        {
            GotFrame,
            EndOfFile,
            Exit
        };
 
        void decodeThread();
        EDecodeFrameResult decodeFrame(AVFrame& frame, int& frameFirstPacketDTS);
        AVPacket* popPacket();
        void clearFrameQueue(std::queue<Frame>& frameQueue, bool emitCallback);
 
    private:
        using FrameQueue = std::queue<Frame>;
 
        Video*                      mVideo;
        AVCodec*                    mCodec = nullptr;
        AVCodecContext*             mCodecContext = nullptr;
 
        int                         mStream = -1;                           
 
        std::thread                 mDecodeThread;                          
        OnClearFrameQueueFunction   mOnClearFrameQueueFunction;             
        bool                        mExitDecodeThreadSignalled = false;     
 
        FrameQueue                  mFrameQueue;                            
        FrameQueue                  mSeekFrameQueue;                        
        FrameQueue*                 mActiveFrameQueue = &mFrameQueue;       
        mutable std::mutex          mFrameQueueMutex;                       
        std::condition_variable     mFrameDataAvailableCondition;           
        std::condition_variable     mFrameQueueRoomAvailableCondition;      
        bool                        mCancelWaitFrameQueueEmpty = false;     
 
        // Producer/consumer variables for packet queue:
        std::queue<AVPacket*>       mPacketQueue;                           
        std::mutex                  mPacketQueueMutex;                      
        std::condition_variable     mPacketAvailableCondition;              
        
        bool                        mFinishedProducingFrames = false;       
 
        std::unique_ptr<AVPacket>   mSeekStartPacket;                       
        std::unique_ptr<AVPacket>   mSeekEndPacket;                         
        std::unique_ptr<AVPacket>   mEndOfFilePacket;                       
        std::unique_ptr<AVPacket>   mIOFinishedPacket;                      
        double                      mSeekTargetSecs;                        
 
        utility::AutoResetEvent     mEndOfFileProcessedEvent;               
        utility::AutoResetEvent     mSeekStartProcessedEvent;               
        utility::AutoResetEvent     mReceiveFrameEvent;                     
        int                         mReceiveFrameResult = 0;                
 
        double                      mLastFramePTSSecs = 0.0;                
        int                         mFrameFirstPacketDTS = -INT_MAX;        
    };
 
 
    class NAPAPI Video final
    {
    public:
        Video(const std::string& path);
 
        // Destructor
        virtual ~Video();
 
        Video(Video&) = delete;
 
        Video& operator=(const Video&) = delete;
 
        Video(Video&&) = delete;
 
        Video& operator=(Video&&) = delete;
 
        virtual bool init(utility::ErrorState& errorState);
 
        Frame update(double deltaTime);
 
        void play(double time = 0.0);
 
        bool isPlaying() const                  { return mPlaying; }
 
        void stop(bool blocking);
 
        void seek(double seconds);
 
        double getCurrentTime() const;
 
        double getDuration() const              { return mDuration; }
 
        int getWidth() const                    { return mWidth; }
 
        int getHeight() const                   { return mHeight; }
 
        const std::string& getPath() const      { return mPath; }
 
        bool hasAudio() const                   { return mAudioState.isValid(); }
 
        bool audioEnabled() const               { return hasAudio() && mDecodeAudio; }
 
        bool        mLoop = false;              
        float       mSpeed = 1.0f;              
        
        nap::Signal<Video&> mDestructedSignal; 
 
    private:
 
        friend class audio::VideoNode;
        friend class AVState;
 
        enum class EProducePacketResult : uint8_t
        {
            GotAudioPacket      = 1,                                                    
            GotVideoPacket      = 2,                                                    
            GotUnknownPacket    = 4,                                                    
            GotPacket           = GotAudioPacket | GotVideoPacket | GotUnknownPacket,   
            EndOfFile           = 8,                                                    
            Error               = 16                                                    
        };
 
        enum class IOThreadState
        {
            WaitingForEOF,              
            Playing,                    
            SeekRequest,                
            SeekingStartFrame,          
            SeekingTargetFrame          
        };
 
        static bool sInitAVState(AVState& destState, const AVStream& stream, AVDictionary*& options, utility::ErrorState& errorState);
 
        void ioThread();
        
        void startIOThread();
 
        EProducePacketResult ProducePacket(AVState* targetState);
 
        void exitIOThread(bool join);
 
        bool getNextAudioFrame(const AudioFormat& targetAudioFormat);
 
        void clearPacketQueue();
 
        void clearFrameQueue();
        
        void onClearVideoFrameQueue();
 
        void onClearAudioFrameQueue();
 
        void setErrorOccurred(const std::string& errorMessage);
 
        void finishSeeking(AVState& seekState, bool shouldDrainFrameQueue);
 
        void setSeekTarget(AVState& seekState, double seekTargetSecs);
 
        void setIOThreadState(IOThreadState threadState);
 
        bool allocatePacket(uint64_t inPacketSize);
 
        void deallocatePacket(uint64_t inPacketSize);
 
        void decodeAudioStream(bool enabled);
 
        bool OnAudioCallback(uint8_t* dataBuffer, int sizeInBytes, const AudioFormat& targetAudioFormat);
 
    private:
        std::string             mPath;                                      
        static const double     sClockMax;
        AVFormatContext*        mFormatContext = nullptr;
        bool                    mPlaying = false;                           
        int                     mWidth = 0;                                 
        int                     mHeight = 0;                                
        float                   mDuration = 0.0f;                           
        double                  mSystemClockSecs = sClockMax;               
        double                  mAudioDecodeClockSecs = -1;                 
        double                  mAudioClockSecs = -1;                       
 
        std::string             mErrorMessage;                              
        
        std::mutex              mTotalPacketQueueSizeLock;                  
        std::condition_variable mPacketQueueRoomAvailableCondition;         
        uint64_t                mTotalPacketQueueSize = 0;                  
 
        AVState                 mVideoState;                                
        AVState                 mAudioState;                                
        AVState*                mSeekState = nullptr;                       
        std::thread             mIOThread;                                  
 
        bool                    mExitIOThreadSignalled = false;             
        
        int64_t                 mSeekKeyframeTarget = -1;                   
        int64_t                 mSeekTarget = -1;                           
        double                  mSeekTargetSecs = 0.0f;                     
 
        SwrContext*             mAudioResampleContext = nullptr;            
        Frame                   mCurrentAudioFrame;                         
        std::vector<uint8_t>    mAudioResampleBuffer;                       
        uint8_t*                mCurrentAudioBuffer = nullptr;              
        uint64_t                mAudioFrameReadOffset = 0;                  
        uint64_t                mAudioFrameSize = 0;                        
        bool                    mDecodeAudio = false;                       
        IOThreadState           mIOThreadState = IOThreadState::Playing;    
    };
}