Decoder.m

//
//  Decoder.m
//  FFmpegPlayTest
//
//  Created by Jack on 11/2/12.
//  Copyright (c) 2012 Jack. All rights reserved.
//

#import <CoreAudio/CoreAudioTypes.h>
#import "Decoder.h"

/// Use another thread to decode audio and add it to
/// the audio buffer
//#define USE_AUDIO_THREAD

//#define _cplusplus

#define kMaxVideoQueueSize (5 * 256 * 1024) // ~ 1MB
#define kMaxAudioQueueSize (5 * 16 * 1024)
#define AUDIO_DIFF_AVG_NB 20
#define SDL_AUDIO_BUFFER_SIZE 1024
#define SAMPLE_CORRECTION_PERCENT_MAX 10.0
#define AV_NOSYNC_THRESHOLD 10.0
#define AV_SYNC_THRESHOLD 0.01
#define AV_SYNC_VIDEO_THRESHOLD 0.01
#define AUDIO_DIFF_AVG_NB 20
#define kDecodeKeyVideoPktPts @"videoPacketPts"

/**
 This function should return TRUE (1) if the decode process is quited
 otherwise return FALSE (0)
 
 Usage: check status to see if "quit signal" has sent, if so return 1 
 to interupt IO process.
 */
static int decodeInteruptCallback(void * ctx) {
    int* status = (int*) ctx;
    return (*status == DecodeStateStopped);
}

double global_video_pkt_pts;
int our_get_buffer(struct AVCodecContext *c, AVFrame *pic) {
    int ret = avcodec_default_get_buffer(c, pic);
    uint64_t *pts = av_malloc(sizeof(uint64_t));
    *pts = global_video_pkt_pts;
    pic->opaque = pts;
    return ret;
}
//---------------------------------------------------------------------
void our_release_buffer(struct AVCodecContext *c, AVFrame *pic) {
    if(pic) av_freep(&pic->opaque);
    avcodec_default_release_buffer(c, pic);
}

#pragma mark - Decoder Extension
@interface Decoder ()
{
}
@end

#pragma mark - Decoder Implementation

@implementation Decoder
@synthesize clockMode=_clockMode;

#pragma mark Init and utilities
- (id) initWithContentURL: (NSURL*) url
{
    self = [super init];
    if (self) {
        [[FFMpegEngine shareInstance] initFFmpegEngine];
        
        // Assign callback to check status of decode process
        AVIOInterruptCB interuptCallback;
        interuptCallback.callback = decodeInteruptCallback;
        interuptCallback.opaque = &decodeStatus;
        decodeStatus = DecodeStateUnknown;

        // Open video file and assign ffmpeg io interupt callback
        _video = [[Video alloc] initWithUrl: url
                             interuptCallback: interuptCallback];
        
        if (!_video) {
            NSLog(@"Failed init decoder");
            return nil;
        }
        
        [_video videoCodecContext]->get_buffer  = our_get_buffer;
        [_video videoCodecContext]->release_buffer = our_release_buffer;
        
        // Using ffmpeg for audio decoding
         decodeAudioMode = DecodeAudioModeFFmpeg;
        //decodeAudioMode = DecodeAudioModeNative; // TEST
        
        
        _clockMode = DecodeMasterClockExternal;
        // For the current version, we force output of decoder
        // to be RGB_565, for the later update this value should
        // be set via a public accessor
        glPixelFormat = GL_UNSIGNED_SHORT_5_6_5;
        
        // [JACK] Test directly render YUV420
//        ffPixelFormat = PIX_FMT_RGB565;
        ffPixelFormat = PIX_FMT_YUV420P;
        
        
        // TODO: remmeber to release swsScale
        // Init software scale context
        swsContext = sws_getContext([_video videoCodecContext]->width,
                                    [_video videoCodecContext]->height,
                                    [_video videoCodecContext]->pix_fmt,
                                    [_video videoCodecContext]->width,
                                    [_video videoCodecContext]->height,
                                    ffPixelFormat,     // match with outputPixelFormat
                                    SWS_FAST_BILINEAR,
                                    NULL, NULL, NULL);
        
        
        if (![self initDecoder])
        {
            NSLog(@"Failed init decoder");
            return nil;
        }
        
        videoOn = YES;
        audioOn = YES;
        decodeStatus |= DecodeStateInited;
    }
    return self;
}


//----------------------------------------------------------------
- (BOOL) initDecoder
{
    BOOL ret = TRUE;
    
    // >> VIDEO <<<
    // Initialize video queue buffer
    _videoPktQueue = [[AVPacketQueue alloc] initWithSize:kMaxVideoQueueSize];
    _videoPicQueue = [[PictureQueue alloc] init]; // using fixed size queue
    
    if(!_videoPicQueue || !_videoPktQueue)
    {
        ret = FALSE;
        goto finish;
    }
    
    startTime = CACurrentMediaTime();
    // Initialize timer variable
    videoFrameTimer = CACurrentMediaTime();
    videoFrameLastDelay = 40e-3;
    videoFrameLastPts = 0;
    
    videoClock = 0;
    videoCurrentPts = 0;
    videoCurrentPtsTime = CACurrentMediaTime();
    
    // >>> AUDIO <<<
    // Initialize audio buffer queue
    _audioPktQueue = [[AVPacketQueue alloc] initWithSize:kMaxAudioQueueSize];
    if (!_audioPktQueue) {
        ret = FALSE;
        goto finish;
    }
    
    UInt32 bitPerChannel   = [FFMpegEngine bitsForSampleFormat: [_video audioCodecContext]->sample_fmt];
    BOOL isNonInterleaved       = NO;
    UInt32 channelsPerSample     = [_video audioCodecContext]->channels;
    Float64 sampleRate          = [_video audioCodecContext]->sample_rate;
    
    UInt32 mBytesPerFrame    = (isNonInterleaved ? 1 : channelsPerSample) * (bitPerChannel/8);
    
    // Init audio buffer for decoded audio frames
    _audioBuffer = [[PlayerAudioBuffer alloc] initBufferForDuration: 1
                                                 bytesPerSample: mBytesPerFrame
                                                     sampleRate: sampleRate];
    
    // Initialize audio clock
    audioClock = 0;
    
    // TODO: define when decoder connect to audio speaker
    audioHWBufferSpec = 0;
    
    // Initialize audio sync accumulator
    audioDiffCum = 0;
    audioDiffAverageCoef = exp(log(0.01 / AUDIO_DIFF_AVG_NB));
    audioDiffAverageCount = 0;
    audioDiffThreshold= 2.0 * SDL_AUDIO_BUFFER_SIZE / [_video audioCodecContext]->sample_rate;
    memset(&pktFromQueue, 0, sizeof(pktFromQueue));
    memset(&pktTemp, 0, sizeof(pktTemp));
    
finish:
    if (!ret){
        NSLog(@"Failed to init decoder");
    }
    else{
        NSLog(@"Finished init decoder");
    }
    
    return ret;
}

//----------------------------------------------------------------
#pragma mark Controllable Protocol
//----------------------------------------------------------------
- (void) pause
{
    //TODO: Implement
    NSLog(@"Decoder Paused");
}
- (void) resume
{
    NSLog(@"Decoder Resumed");
}
//----------------------------------------------------------------
- (BOOL) isReady
{
    return ((decodeStatus & DecodeStateInited) | (decodeStatus & DecodeStatePaused)) != 0 ;
}
//----------------------------------------------------------------
- (void) start
{
    
    //[self performSelectorInBackground:@selector(demuxVideo:) withObject:self];
    demuxThread = [[NSThread alloc] initWithTarget:self
                                          selector:@selector(demuxVideo:)
                                            object:self];

    decodeVideoThread = [[NSThread alloc] initWithTarget:self
                                                selector:@selector(decodeVideo:)
                                                  object:self];
    
    decodeAudioThread = [[NSThread alloc] initWithTarget:self
                                                selector:@selector(decodeAudio:)
                                                  object:self];
    
    [demuxThread start];

    if (videoOn) {
        [decodeVideoThread start];
    }
    
    if (audioOn) {
        [decodeAudioThread start];
    }

    startTime = CACurrentMediaTime();;
    decodeStatus = DecodeStateDecoding;
    NSLog(@"Decoder started");
}
//----------------------------------------------------------------
- (void) stop
{
    //TODO: Implement
    if (demuxThread) {
        [demuxThread cancel];
    }
    
    if (decodeVideoThread && [decodeVideoThread isExecuting]) {
        [decodeVideoThread cancel];
    }
    if (decodeAudioThread && [decodeAudioThread isExecuting]) {
        [decodeAudioThread cancel];
    }
    
    NSLog(@"Decoder Stopped");
}
//----------------------------------------------------------------
- (BOOL) isRuning
{
    //TODO: Implement
    return NO;
}

#pragma mark Decoding Threads

- (void) demuxVideo: (id) data
{
    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
       
        NSThread* currentThread = [NSThread currentThread];
        AVPacket readPkt, *pReadPkt = &readPkt;
        AVFormatContext* pContext = [_video formatContext];
        for (;;) {
            /// Check if thread is cancelled, OR ***decode finish***
            if ([currentThread isCancelled]) {
                NSLog(@"Demux thread cancelled");
                // TODO: release resource and allocated memory here
                break;
            }
            
            // TODO: Handle seeking stuff here
            // flush queue, push the flush pkt to queue
            
            
            // Check if pkt queue data size is exceed the limit,
            // if so delay, than continue to next loop
//            if ([_videoPktQueue dataSize] > kMaxVideoQueueSize||
//                [_audioPktQueue dataSize] > kMaxAudioQueueSize)
//            {
//                NSLog(@"Waiting for space in packet queue");
//                usleep(10000); // delay for 10 miliseconds
//                continue;
//            }
 
            if (av_read_frame(pContext, pReadPkt) < 0) {
                if (&pContext->pb && &pContext->pb->error) {
                    // NO Error, wait for user input to interrupt.
                    usleep(10000);
                    continue;
                }else{
                    break;
                }
            }
            
            // Push packet into corresponding queue
            if (pReadPkt->stream_index == [_video videoStreamIndex]) {
                if(videoOn){
                    NSLog(@"Push packet to video queue");
                    //NSLog(@"V---");
                    [_videoPktQueue pushPacket:pReadPkt blocked:YES];
                }else{
                    av_free_packet(pReadPkt);
                }
            }else if(pReadPkt->stream_index == [_video audioStreamIndex]){
                if (audioOn) {
                    NSLog(@"Push packet to audio queue");
                    //NSLog(@"---A");
                    //NSLog(@"Pkt'size: %d, duration: %d, dts: %lld", pReadPkt->size, pReadPkt->duration, pReadPkt->pts);
                    [_audioPktQueue pushPacket:pReadPkt blocked:YES];
                }else{
                    av_free_packet(pReadPkt);
                }
            }else{
                av_free_packet(pReadPkt);
            }
        }
        
        /// ???: Why do have to wait for cancelled signal, why dont just finish
        /// the thread routine?
        // Wait for the quit signal
        while (![currentThread isCancelled]) {
            usleep(100000);
        }
    [pool release];
    NSLog(@"Demux thread finished");
}

//----------------------------------------------------------------
- (void) decodeVideo: (id) data
{
    NSLog(@"Decoding video stream");
    
    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];

        
        AVCodecContext* pVideoCodecCtx  = [_video videoCodecContext];
        NSThread* currentThread = [NSThread currentThread];
        
        AVPacket readPkt, *pReadPkt = &readPkt;
        AVFrame* pFrame;
        int frameFinished;
        NSTimeInterval pts;
        int videoWidth = [_video videoCodecContext]->width;
        int videoHeight = [_video videoCodecContext]->height;
        
        NSLog(@"Video width: %d, height: %d", videoWidth, videoHeight);
        
        pFrame = avcodec_alloc_frame();
        
        
        for (;;) {
            /// Check if thread is cancelled
            if ([currentThread isCancelled]) {
                NSLog(@"Decode video thread cancelled");
                break;
            }
            
            /// Get pkt from queue
            if (![_videoPktQueue popPacket:pReadPkt blocked:YES]) {
                NSLog(@"Get no packet from video queue");
                continue;
            }
            
            /* TODO: Handle seeking packet
            if(packet->data == flush_pkt.data) {
                avcodec_flush_buffers(vidState->video_st->codec);
                continue;
            }
            */
            
            pts = 0;
            global_video_pkt_pts = pReadPkt->pts;
            
            // Decode video frame, memory will be allocate automatically for pFrame
            avcodec_decode_video2(pVideoCodecCtx, pFrame, &frameFinished, pReadPkt);

            if(pReadPkt->dts == AV_NOPTS_VALUE
               && pFrame->opaque && *(uint64_t*)pFrame->opaque != AV_NOPTS_VALUE) {
                pts = *(uint64_t *)pFrame->opaque;
            } else if(pReadPkt->dts != AV_NOPTS_VALUE) {
                pts = pReadPkt->dts;
            } else {
                pts = 0;
            }


            pts *= av_q2d([_video videoStream]->time_base);

            //NSLog(@"Picture pts before sync: %f", pts);
            // Decoded a frame?
            if (frameFinished) {
                NSLog(@"Got a frame");
                
                // [TEST] Dont adjust video pts
                pts = [self synchronizeVideoFrame:pFrame
                                         framePts:pts];
                
                //NSLog(@"Picture pts after sync: %f", pts);

                // Reuse picture in the queue
                VideoPicture* picture = [_videoPicQueue pictureToWriteWithBlock:YES];
                
                // If it NULL, reallocate the new one
                if (!picture) {
                    picture = [[VideoPicture alloc] initWithPixelFormat:ffPixelFormat
                                                                    width:videoWidth
                                                                   height:videoHeight];
                }
                

//                NSLog(@"Source video pix format: %d", [_video videoCodecContext]->pix_fmt);

                
                /// Convert frame data format to picture's format
                // get AVPicture instance of the picture which using the same
                // picture data
//                AVPicture* avPict  = [picture avPicture];
//                NSLog(@"avPict data: %p", avPict->data);
//                NSLog(@"avPict data 0: %p", avPict->data[0]);
//                NSLog(@"avPict data 1: %p", avPict->data[1]);
//                NSLog(@"avPict data 2: %p", avPict->data[2]);
                
                /// [JACK] Test no sws pixel convert
//                sws_scale(swsContext,
//                          (const uint8_t * const *)pFrame->data,
//                          pFrame->linesize,
//                          0,
//                          videoHeight,
//                          avPict->data,
//                          avPict->linesize);
                av_picture_copy([picture avPicture],
                                (const AVPicture*) pFrame, 
                                [_video videoCodecContext]->pix_fmt,
                                videoWidth,
                                videoHeight);
                
                // Update picture pts
                [picture setPts:pts];
                
                /// Push frame to picture queue
                [_videoPicQueue pushPicture:picture blockingMode:YES];
            }
            
            av_free_packet(pReadPkt);
            
        }
        av_free(pFrame);
        NSLog(@"Decode video finished");
        while (![currentThread isCancelled]) {
            usleep(100000);
        }
    [pool release];
}

//----------------------------------------------------------------
/**
 Right now the audio is decoded immediately when it's requested from
 audio handler, so decodeAudio:data virtually does nothing. The code 
 is kept for reference.
 */
- (void) decodeAudio: (id) data
{
#ifdef USE_AUDIO_THREAD
    
    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
        AVCodecContext* pAudioCodecCtx  = [_video audioCodecContext];
        NSThread* currentThread = [NSThread currentThread];
        
        
        int decodedLen, decodedFrameSize;
        
        AVPacket readPkt;
        AVPacket *pReadPkt = &readPkt;
        
        AVPacket pktTemp;
        AVPacket *pPktTemp = &pktTemp; // hold data that is being process.
        
        AVFrame  *pDecodedFrame;
        pDecodedFrame = avcodec_alloc_frame();
        avcodec_get_frame_defaults(pDecodedFrame);
        
        // Zero out temporary packet
        memset(pPktTemp, 0, sizeof(AVPacket));
        memset(pReadPkt, 0, sizeof(AVPacket));
        
        
        // Loop that decode audio packets and push decode data to audio buffer
        JLogAudio(@"Decoding audio stream");
        for (;;) {
            
            
            // Check if thread is cancelled
            if ([currentThread isCancelled]) {
                NSLog(@"Decode audio thread cancelled");
                break;
            }
            
            // A packet may contain more than one frame, so to decode a packet
            // we need a loop
            while(pktTemp.size > 0){
                
                int gotFrame = 0;
                //avcodec_get_frame_defaults(&decodedFrame);
                
                // Decode audio data to decoded frame
                decodedLen = avcodec_decode_audio4(pAudioCodecCtx,
                                                   pDecodedFrame,
                                                   &gotFrame,
                                                   pPktTemp);
                // If error occur, we skip the packet
                if (decodedLen < 0) {
                    pPktTemp->size = 0;
                    break;
                }
                
                // Update pointer and length in temp packet
                pPktTemp->data += decodedLen;
                pPktTemp->size -= decodedLen;
                
                // If a frame is found
                if (gotFrame) {
                    
                    // Calculate size in bytes for decoded frames
                    decodedFrameSize = av_samples_get_buffer_size(NULL, pAudioCodecCtx->channels,
                                                                  pDecodedFrame->nb_samples,
                                                                  pAudioCodecCtx->sample_fmt, 1);
                    
                    /// FIXME: this push will be call so frequently so it will block access
                    /// to the audio buffer so often that will flicking noise in audio output
                    // Push those data to buffer
                    [_audioBuffer pushSample:pDecodedFrame->data[0]
                                        size:decodedFrameSize
                                     blocked:YES];
                }
            };
            
            // Free packet after decode it
            if(pReadPkt->data)
                av_free_packet(pReadPkt);
            
            // Get next pkt from queue
            if (![_audioPktQueue popPacket:pReadPkt blocked:YES]) {
                NSLog(@"Get no packet from audio queue");
                continue;
            }
            
            
            // TODO: Handleflush packets
            //if(pkt->data == flush_pkt.data) {
            //    avcodec_flush_buffers(vidState->audio_st->codec);
            //    continue;
            //}
            
            // Assign data to temporary packet
            pktTemp.data = pReadPkt->data;
            pktTemp.size = pReadPkt->size;
            
            /* if update, update the audio clock w/pts */
            //if(pkt->pts != AV_NOPTS_VALUE) {
            //    vidState->audio_clock = av_q2d(vidState->audio_st->time_base) * pkt->pts;
            //}        
        }
        
        avcodec_free_frame(&pDecodedFrame);
    }
    [pool release];
    return;
#endif
}
//----------------------------------------------------------------
#pragma mark - Utilities Methods
- (NSTimeInterval) masterClock
{
    if (_clockMode == DecodeMasterClockVideo) {
        return [self videoClock];
    }else if (_clockMode == DecodeMasterClockAudio){
        return [self audioClock];
    }else{
        return [self externalClock];
    }
}
//-----------------------------------------------------------------
- (NSTimeInterval) audioClock
{
    double pts;
    int hw_buf_size, bytes_per_sec, n;
    
    pts = audioClock; /* maintained in the audio thread */
    hw_buf_size = audioBufSize - audioBufIndex;
    bytes_per_sec = 0;
    n = [_video audioCodecContext]->channels * 2;
    if([_video audioStream]) {
        bytes_per_sec = [_video audioCodecContext]->sample_rate * n;
    }
    if(bytes_per_sec) {
        pts -= (double)hw_buf_size / bytes_per_sec;
    }
    return pts;
}
//-----------------------------------------------------------------
- (NSTimeInterval) videoClock
{
    double delta;
    
    delta = CACurrentMediaTime() - videoCurrentPtsTime;
    return videoCurrentPts + delta;
}
//-----------------------------------------------------------------
- (NSTimeInterval) externalClock
{
    return CACurrentMediaTime() -startTime;
}
//-----------------------------------------------------------------
- (NSTimeInterval) synchronizeVideoFrame: (AVFrame*) srcFrame
                                framePts: (NSTimeInterval) pts
{
    
    NSTimeInterval frameDelay;
    
    if(pts != 0) {
        /* if we have pts, set video clock to it */
        videoClock = pts;
    } else {
        /* if we aren't given a pts, set it to the clock */
        pts = videoClock;
    }
    /* update the video clock */
    frameDelay = av_q2d([_video videoStream]->codec->time_base);
    /* if we are repeating a frame, adjust clock accordingly */
    frameDelay += srcFrame->repeat_pict * (frameDelay * 0.5);
    videoClock += frameDelay;
    return pts;
}

//----------------------------------------------------------------
/**
 decode audio to audio buffer and store audio of first sample of 
 new decoded audio sample
 */
- (int) decodeAudioOutBuff: (NSTimeInterval*) pts
{
    AVCodecContext* pAudioCodecCtx  = [_video audioCodecContext];
    AVStream* pAudioStream = [_video audioStream];
    
    int decodedLen =0 , decodedFrameSize =0 , n = 0;
    
    AVFrame  *pDecodedFrame = nil;
    pDecodedFrame = avcodec_alloc_frame();
    avcodec_get_frame_defaults(pDecodedFrame);
    
    // Loop that decode audio packets and push decode data to audio buffer
    NSLog(@"Decoding audio stream");
    for (;;) {
        
        // A packet may contain more than one frame, so to decode a packet
        // we need a loop
        while(pktTemp.size > 0){
            
            int gotFrame = 0;
            //avcodec_get_frame_defaults(&decodedFrame);
            
            // Decode audio data to decoded frame
            decodedLen = avcodec_decode_audio4(pAudioCodecCtx,
                                               pDecodedFrame,
                                               &gotFrame,
                                               &pktTemp);
            // If error occur, we skip the packet
            if (decodedLen < 0) {
                pktTemp.size = 0;
                break;
            }
            
            // Update pointer and length in temp packet
            pktTemp.data += decodedLen;
            pktTemp.size -= decodedLen;
            
            // If a frame is found
            if (gotFrame) {
                
                // Calculate size in bytes for decoded frames
                decodedFrameSize = av_samples_get_buffer_size(NULL, pAudioCodecCtx->channels,
                                                              pDecodedFrame->nb_samples,
                                                              pAudioCodecCtx->sample_fmt, 1);
                
                // we got decode data pointed by decodedFrame.data[0], with size
                // decodedFrameSize
                memcpy(audioBuffer, pDecodedFrame->data[0], decodedFrameSize);
            }else{
                decodedFrameSize = 0;
            }
            
            if (decodedFrameSize <= 0) {
                continue;
            }
            *pts = audioClock;
            n = 2 * pAudioCodecCtx->channels; // 16bits per sample per channel
            audioClock += (double)decodedFrameSize / (double)(n * pAudioCodecCtx->sample_rate);
            
            // decodedFrameSize > 0;
            break;
        }
        
        // If got data, break and return
        if (decodedFrameSize > 0) {
            break;
        }

        // Free packet after decode it
        if(pktFromQueue.data)
            av_free_packet(&pktFromQueue);
        
        // TODO: detect quit signal
        
        // Get next pkt from queue
        if (![_audioPktQueue popPacket:&pktFromQueue blocked:YES]) {
            NSLog(@"Get no packet from audio queue");
            return -1;
        }
        
        
        // TODO: Handleflush packets
        //if(pkt->data == flush_pkt.data) {
        //    avcodec_flush_buffers(vidState->audio_st->codec);
        //    continue;
        //}
        
        // Assign data to temporary packet
        pktTemp.data = pktFromQueue.data;
        pktTemp.size = pktFromQueue.size;
        
        /* if update, update the audio clock w/pts */
        if(pktFromQueue.pts != AV_NOPTS_VALUE) {
            audioClock = av_q2d(pAudioStream->time_base) * pktFromQueue.pts;
        }
    }
    
    avcodec_free_frame(&pDecodedFrame);
    return decodedFrameSize;
}
//----------------------------------------------------------------
- (int) synchronizeAudioBuff: (int16_t *) pBuf
                      OfSize: (int) auSize
                         pts: (double) pts
{
    AVCodecContext* pAudioCodecCtx  = [_video audioCodecContext];
    int n = 2 * pAudioCodecCtx->channels;
    double refClock;
    double diff, avg_diff;
    int wanted_size, min_size, max_size;
    
    
    if (_clockMode != DecodeMasterClockAudio) {
   
        refClock = [self masterClock];
        diff = [self audioClock] - refClock;
        NSLog(@"A-Ref diff: %f", diff);
/*
        if (diff < AV_NOSYNC_THRESHOLD) {
            audioDiffCum = diff + audioDiffAverageCoef * audioDiffCum;
            
            NSLog(@"DiffCum: %f, avgCount: %d", audioDiffCum, audioDiffAverageCount);
            
            if (audioDiffAverageCount < AUDIO_DIFF_AVG_NB) {
                audioDiffAverageCount++;
            }else{
                avg_diff = audioDiffCum * (1.0 - audioDiffAverageCoef);
                NSLog(@"New avg: %f, threshold: %f", avg_diff, audioDiffThreshold);
                
                if (fabs(avg_diff) >= audioDiffThreshold) {
                    
                    wanted_size = auSize + ((int)(diff * pAudioCodecCtx->sample_rate) * n);

                    min_size = (float) auSize * ((100.0 - SAMPLE_CORRECTION_PERCENT_MAX) / 100.0);
                    max_size = (float) auSize * ((100.0 + SAMPLE_CORRECTION_PERCENT_MAX) / 100.0);

                    NSLog(@"wanted size: %d, max: %d, min: %d", wanted_size, max_size, min_size);
                    if (wanted_size < min_size)
                    {
                        wanted_size = min_size;
                    }
                    else if(wanted_size > max_size)
                    {
                        wanted_size = max_size;
                    }
                    else if (wanted_size > auSize)
                    {
                        uint8_t *sampleEnd, *q;
                        int nb;
                        
                        nb = auSize - wanted_size;
                        sampleEnd = (uint8_t*) pBuf + auSize - n;
                        q = sampleEnd + n;
                        while(nb > 0) {
                            memcpy(q, sampleEnd, n);
                            q += n;
                            nb -= n;
                        }
                        auSize = wanted_size;
                    }
                }
             }
 
        }else{
            audioDiffAverageCount = 0;
            audioDiffCum = 0;
        }
 */
    }
    NSLog(@"Return auSize: %d", auSize);
    return auSize;
}
//----------------------------------------------------------------
/**
 Decode audio samples to pBuffer, given size of buffer and pts of the
 buffer
 @param pBuffer Pointer to the buffer
 @param capacity size of buffer
 @param pts Presenting timestamp of the buffer
 
 @return return filled size
 */
- (void) fillPCMAudioIntoBuff:(void *) pBuffer
                  capacity:(UInt32) capacity
{
    int needRead = capacity;
    int audioSize, copySize;
    double pts;
    while (needRead > 0) {
        // if there is no samples in audio buffer, decode new samples
        if(audioBufIndex >= audioBufSize){
            audioSize = [self decodeAudioOutBuff:&pts];
            //NSLog(@"Audio size before: %d", audioSize);
            if (audioSize < 0) {
                audioBufSize = 1024;
                memset(audioBuffer, 0, audioBufSize);
            }else{
//                audioSize = [self synchronizeAudioBuff: (int16_t*)audioBuffer
//                                                OfSize: audioSize
//                                                   pts: pts];
                //NSLog(@"Audio size after: %d", audioSize);
                audioBufSize = audioSize;
            }
            audioBufIndex = 0;
        }
        
        
        copySize = audioBufSize - audioBufIndex;
        if (copySize > needRead) {
            copySize = needRead;
        }
        memcpy(pBuffer, audioBuffer + audioBufIndex, copySize);
        needRead -= copySize;
        pBuffer += copySize;
        audioBufIndex += copySize;
    }
}

//----------------------------------------------------------------
- (void) fillAudioPktToBuff: (AudioQueueBufferRef)aqBuffer
                     forPts: (NSTimeInterval) pts
{
    int ret = 0;
    while ((aqBuffer->mAudioDataByteSize < aqBuffer->mAudioDataBytesCapacity) &&
            aqBuffer->mPacketDescriptionCount < aqBuffer->mPacketDescriptionCapacity)
    {
        ret = [_audioPktQueue popPacket:&pktFromQueue blocked:YES];
        if (ret == 0) {
            break;
        }
        memcpy(aqBuffer->mAudioData + aqBuffer->mAudioDataByteSize, pktFromQueue.data, pktFromQueue.size);
        aqBuffer->mPacketDescriptions[aqBuffer->mPacketDescriptionCount].mStartOffset = aqBuffer->mAudioDataByteSize;
        aqBuffer->mPacketDescriptions[aqBuffer->mPacketDescriptionCount].mDataByteSize = pktFromQueue.size;
        aqBuffer->mPacketDescriptions[aqBuffer->mPacketDescriptionCount].mVariableFramesInPacket = [_video audioCodecContext]->frame_size; // TODO: need to calculate it, 0 mean constant frame in packet
        aqBuffer->mPacketDescriptionCount++;
        aqBuffer->mAudioDataByteSize += pktFromQueue.size;
    }
}
//----------------------------------------------------------------
#pragma mark - VideoScreenSource Protocol
- (void) finishFrameForScreen: (VideoScreen*) screen
{
    [_videoPicQueue popPictureWithBlockingMode:YES];
}

//----------------------------------------------------------------
- (CGSize) videoFrameSize
{
    if (!_video) {
        return CGSizeMake(0, 0);
    }
    
    return CGSizeMake([_video videoCodecContext]->width,
                      [_video videoCodecContext]->height);
}

//----------------------------------------------------------------
- (int) pixelFormat
{
    return glPixelFormat;
}

//----------------------------------------------------------------
/**
 This function called by Video screen object get the next video 
 picture to display.
 
 @param screen screen that call this function
 @param lastPts last present time stamp (since start time) of last shown frame
 
 @return picture to render
 */
-(const VideoPicture* const) getPictureForScreen: (VideoScreen*) screen
                                       screenClock:(NSTimeInterval)scrPts
{

 // const VideoPicture* const picture = [_videoPicQueue pictureToReadWithBlock:NO];

//    double actual_delay, delay, sync_threshold;
//    delay = videoCurrentPts - videoFrameLastPts;
//    if (delay <=0 || delay >= 1) {
//        delay = videoFrameLastDelay;
//    }
    
//    videoFrameLastDelay = delay;
    double diff, ref_clock, aclk;
    const VideoPicture* picture = nil;
    
    ref_clock = [self masterClock];
    aclk = [self audioClock];
    //ref_clock = scrPts;
    for(;;){
        picture = [_videoPicQueue pictureToReadWithBlock:YES];
        if(_clockMode  != DecodeMasterClockVideo ) {
            diff = [picture pts] - scrPts ;
            //NSLog(@"Delay: %f, rendering time: %f",diff, [screen renderingTime]);
//            NSLog(@"video picture: pts: %f; ref_clk: %f; diff: %f, videoclk: %f, audioClk: %f, exClk: %f",
//                    [picture pts],ref_clock,diff, [self videoClock],[self audioClock], [self externalClock]);
            NSLog(@"vpts-aclk: %f, vpts-ref-clk: %f, refClk: %f, aclk: %f",[picture pts]-aclk, [picture pts] - ref_clock, ref_clock, aclk);
            
            // TEST, just break here, so video will ouput at max speed
            //NSLog(@"Packet size %d, pic queue: %d", [_videoPktQueue count],[_videoPicQueue size]);

            //break;

            if (diff > AV_SYNC_VIDEO_THRESHOLD){
                usleep(diff* 1000000.0);
                break;
            }
            else if (diff < (-AV_SYNC_VIDEO_THRESHOLD))
            {
                if(![_videoPicQueue size])
                    continue;
                break;
            }
            break;
        }
    }

    videoFrameLastPts = videoCurrentPts;
    videoCurrentPts = [picture pts];
    videoCurrentPtsTime = CACurrentMediaTime();

    return picture;
}

//----------------------------------------------------------------
#pragma mark - AudioQueueSource Protocol
/**
 There are two options for audio decoding: (1) using ffmpeg as external soft 
 decoder, (2) using native ios decoder (nativ and soft).
 
 (1) If ffmpeg is used as decoder, AudioStreamBasicDescription (ASBD) should be 
 formated as uncompressed LPCM format.
 
 (2) If ios decoder is used, ASBD should be formated to conform the codec of 
 audio strem (often used: mp3, and aac)
 
 In this current version, we use ffmpeg as audio decoder, so ASBD is filled as
 uncompressed LPCM.
 */
// TODO: Need improvment
//  Implement fillAudioStream for general case that includes iOS decoder (AAC, MP3).
//  Check output from ffmpeg for this value
- (BOOL) fillAudioStreamDescription: (AudioStreamBasicDescription*) pASBD
{
    if (!_video){
        NSLog(@"Video has not loaded yet!");
        return FALSE;
    }
    
    if (decodeAudioMode == DecodeAudioModeFFmpeg) {
        /**
         Fill ASBD for LPCM case (ref: FillOutASBDForLPCM inline function in CoreAudioTypes
         
         */
        UInt32 validBitPerChannel   = [FFMpegEngine bitsForSampleFormat: [_video audioCodecContext]->sample_fmt];
        UInt32 totalBitPerChannel   = validBitPerChannel;
        

        BOOL isFloat                = NO;
        BOOL isBigEndian            = NO;
        BOOL isNonInterleaved       = NO;
        
        UInt32 channelsPerFrame     = [_video audioCodecContext]->channels;
        Float64 sampleRate          = [_video audioCodecContext]->sample_rate;
        
        pASBD->mSampleRate       = sampleRate;
        pASBD->mFormatID         = kAudioFormatLinearPCM;
        
        pASBD->mFormatFlags      =
        (isFloat ? kAudioFormatFlagIsFloat : kAudioFormatFlagIsSignedInteger) |
        (isBigEndian ? ((UInt32)kAudioFormatFlagIsBigEndian) : 0) |
        ((!isFloat && (validBitPerChannel == totalBitPerChannel)) ? kAudioFormatFlagIsPacked : kAudioFormatFlagIsAlignedHigh)|
        (isNonInterleaved ? ((UInt32)kAudioFormatFlagIsNonInterleaved) : 0);
        
        pASBD->mBytesPerPacket   = (isNonInterleaved ? 1 : channelsPerFrame) * (totalBitPerChannel/8);
        pASBD->mFramesPerPacket  = 1;
        
        pASBD->mBytesPerFrame    = (isNonInterleaved ? 1 : channelsPerFrame) * (totalBitPerChannel/8);
        pASBD->mChannelsPerFrame = channelsPerFrame;
        pASBD->mBitsPerChannel   = validBitPerChannel;
        pASBD->mReserved         = 0;
        
    }else{ // decodeAudioMode == FFDecodeAudioNative
        
        pASBD->mSampleRate = [_video audioCodecContext]->sample_rate;
        pASBD->mFormatID = kAudioFormatMPEG4AAC;
        pASBD->mFormatFlags = kMPEG4Object_AAC_Main;
        
        // It's VBR, dont need to set these value
        pASBD->mBytesPerPacket = 0;
        pASBD->mFramesPerPacket = [_video audioCodecContext]->frame_size;
        
        
        pASBD->mBitsPerChannel = [FFMpegEngine bitsForSampleFormat: [_video audioCodecContext]->sample_fmt];
        pASBD->mChannelsPerFrame = [_video audioCodecContext]->channels;
        //pASBD->mBytesPerFrame = pASBD->mChannelsPerFrame * pASBD->mBitsPerChannel / 8;
        pASBD->mBytesPerFrame = 0;
        
    }
    
    return TRUE;
}

//----------------------------------------------------------------
/**
 In case of LCPM, p1 packet contain 1 frame so maximum packet size is size of 
 one frame. 
 */
// TODO: Need improvment
//  - Implement fillAudioStream for general case that includes iOS decoder (AAC, MP3).
//  - Check output from ffmpeg for interleave value
- (UInt32) maxAudioPacketSize
{
     if (decodeAudioMode == DecodeAudioModeFFmpeg) {
         UInt32 channelsPerFrame     = [_video audioCodecContext]->channels;
         BOOL isNonInterleaved       = NO;
         UInt32 totalBitPerChannel   = [FFMpegEngine bitsForSampleFormat: [_video audioCodecContext]->sample_fmt];
         
         return (isNonInterleaved ? 1 : channelsPerFrame) * (totalBitPerChannel/8);
     }else{
         return [_video audioCodecContext]->frame_size;
         //return 300;
     }
}

//----------------------------------------------------------------
/**
 Render audio buffer with output data from decoder
 @param aqBuffer the buffer that decode need to fill audio samples in
 @param pts the time reference, indicate what set of sample should be filled in
 */
// TODO: Implement for both ffmpeg-decoder and iOS decoder
//#define USE_AUDIO_THREAD
- (void) renderAudioBuffer:(AudioQueueBufferRef) aqBuffer
                    forPts:(NSTimeInterval)pts
{
    //JLog(@"Rendering AQ buffer");
    if (decodeAudioMode == DecodeAudioModeFFmpeg) {
        
/*
        // Generate sine noise sound for test
        double amplitude = 0.25 * 0x3FFF;
        for (int n = 0; n < aqBuffer->mAudioDataBytesCapacity; n+=2)
        {
            SInt16* pSample = (aqBuffer->mAudioData  + n);
            *pSample = (short)(amplitude * ((double)rand()/(double)(RAND_MAX - 1)));
        }
        aqBuffer->mAudioDataByteSize = aqBuffer->mAudioDataBytesCapacity;
//*/

#ifdef USE_AUDIO_THREAD
        NSTimeInterval bufferPts;
        int readBytes = [_audioBuffer popSampleTo:aqBuffer->mAudioData
                             size:aqBuffer->mAudioDataBytesCapacity
                           outPts:&bufferPts
                          blocked:YES];
        
        aqBuffer->mAudioDataByteSize = readBytes;
        NSLog(@"Fill audio buffer");
#else
        [self fillPCMAudioIntoBuff:aqBuffer->mAudioData
                       capacity:aqBuffer->mAudioDataBytesCapacity];
        aqBuffer->mAudioDataByteSize = aqBuffer->mAudioDataBytesCapacity;
        
#endif
        
    }else{// decodeAudioMode == FFDecodeAudioNative
        // TODO: Implementation
        [self fillAudioPktToBuff: aqBuffer forPts: (NSTimeInterval) pts];
    }
}


- (int) sampleRate
{
    return [_video audioCodecContext]->sample_rate;
}


@end

//----------------------------------------------------------------
#pragma mark - Decoder's Video Properties
@implementation Decoder (VideoProperties)
@dynamic mediaSourceType;

- (int) mediaSourceType
{
    return 0;
}

- (void) setMediaSourceType:(int)mediaSourceType
{
    
}

- (double) currentPlaybackTime
{
    return  0;
}

- (float) currentPlaybackRate
{
    return  0;
}

- (int) mediaTypes
{
    return  0;
}

- (float) duration
{
    return  0;
}

- (float) playableDuration
{
    return  0;
}

// CGSizeZero if not known/applicable.
- (CGSize) videoSize
{
    return CGSizeMake(0, 0);
}

// return NaN indicates the natural start time
- (float) startTime
{
    return NAN;
}

// return NaN indicates the natural end time
- (float) endTime
{
    return NAN;
}


@end