Synchronizing Narration with Video Length, Tone and Frame Content Contextually Rich and Dynamic Storytelling

				
					# Import necessary libraries
from IPython.display import display, Image, Audio
import cv2
import base64
import time
from openai import OpenAI
import os
import requests

# Initialize OpenAI client
def initialize_openai_api(api_key):
    """
    Initializes the OpenAI API client with the provided API key.
    
    Parameters:
    api_key (str): Your OpenAI API key.
    
    Returns:
    OpenAI: An instance of the OpenAI client.
    """
    os.environ["OPENAI_API_KEY"] = api_key
    return OpenAI()

# Function to read and process the video
def read_video(video_path, frame_interval):
    """
    Reads a video file and extracts frames at specified intervals.
    
    Parameters:
    video_path (str): The path to the video file.
    frame_interval (int): The interval at which frames are extracted, e.g. 60 means every 60th frame is extracted.
    
    Returns:
    tuple: A tuple containing a list of base64-encoded frames and the duration of the video. 
    """
    video = cv2.VideoCapture(video_path)
    frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
    frame_rate = video.get(cv2.CAP_PROP_FPS)
    duration = frame_count / frame_rate
    base64Frames = []

    while video.isOpened():
        success, frame = video.read()
        if not success:
            break
        _, buffer = cv2.imencode(".jpg", frame)
        base64Frames.append(base64.b64encode(buffer).decode("utf-8"))

    video.release()
    return base64Frames[::frame_interval], duration

# Function to display frames (Optional)
def display_frames(frames):
    """
    Displays each frame from a list of base64-encoded frames every 0.025 seconds.
    
    Parameters:
    frames (list): A list of base64-encoded frames.
    """
    display_handle = display(None, display_id=True)
    for img in frames:
        display_handle.update(Image(data=base64.b64decode(img.encode("utf-8"))))
        time.sleep(0.025)

# Function to generate narration script using GPT with Vision
def generate_gpt_vision_prompt(frames, prompt_text, max_tokens):
    """
    Generates a narration script using GPT with Vision.
    
    Parameters:
    frames (list): A list of base64-encoded frames.
    prompt_text (str): The prompt text to guide the AI model.
    max_tokens (int): The maximum number of tokens in the generated text.
    
    Returns:
    str: The generated narration script.
    """
    PROMPT_MESSAGES = [
        {
            "role": "user",
            "content": [
                prompt_text,
                *map(lambda x: {"image": x, "resize": 768}, frames),
            ],
        },
    ]
    params = {
        "model": "gpt-4-vision-preview",
        "messages": PROMPT_MESSAGES,
        "max_tokens": max_tokens,
    }
    client = OpenAI()
    result = client.chat.completions.create(**params)
    return result.choices[0].message.content

# Function to convert text to speech
def text_to_speech(text, voice, speaking_rate):
    """
    Converts the provided text to speech using OpenAI's Text-to-Speech API.
    
    Parameters:
    text (str): The text to be converted into speech.
    voice (str): The voice model to be used.
    speaking_rate (float): The rate of speech.
    
    Returns:
    bytes: The audio data of the spoken text.
    """
    response = requests.post(
        "https://api.openai.com/v1/audio/speech",
        headers={
            "Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}",
        },
        json={
            "model": "tts-1-1106",
            "input": text,
            "voice": voice,
            "speaking_rate": speaking_rate,
        },
    )
    audio = b""
    for chunk in response.iter_content(chunk_size=1024 * 1024):
        audio += chunk
    return audio