async def handle_large_stream_chunks(stream: aiohttp.StreamReader, max_buffer_size: int = CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE):
    """
    Handle large stream chunks for streaming responses.
    Breaks large lines into manageable chunks without skipping any data.
    
    :param stream: The stream reader to handle.
    :param max_buffer_size: The maximum size of each chunk to yield.
    :return: An async generator yielding chunks of the stream.
    """
    buffer = bytearray()
    
    async for chunk, _ in stream.iter_chunks():
        if not chunk:
            continue
            
        # Add new chunk to buffer
        buffer.extend(chunk)
        
        # Process all complete lines
        while b'\n' in buffer:
            # Find first newline position
            line_end = buffer.find(b'\n')
            line = buffer[:line_end]
            
            # Process line in chunks if it's too large
            if len(line) > max_buffer_size:
                # Break large line into chunks
                for i in range(0, len(line), max_buffer_size):
                    yield line[i:i + max_buffer_size]
            else:
                yield line
            
            # Remove processed line from buffer
            buffer = buffer[line_end + 1:]
            
    # Yield remaining data if any
    if buffer:
        yield buffer

async def handle_large_stream_chunks(stream: aiohttp.StreamReader, max_buffer_size: int = CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE):
    """
    Handle large stream chunks without storing entire large lines in memory.
    
    :param stream: The stream reader to handle.
    :param max_buffer_size: The maximum size of each chunk to yield.
    :return: An async generator yielding chunks of the stream.
    """
    async for chunk, _ in stream.iter_chunks():
        if not chunk:
            continue
            
        # Process each chunk directly without storing in a buffer
        lines = chunk.split(b'\n')
        
        for line in lines[:-1]:
            # Process complete lines
            if len(line) > max_buffer_size:
                # Break large line into chunks
                for i in range(0, len(line), max_buffer_size):
                    yield line[i:i + max_buffer_size]
            else:
                yield line
                
        # Handle last partial line if exists
        if lines and lines[-1]:
            yield lines[-1]

async def handle_large_stream_chunks(stream: aiohttp.StreamReader, 
                                    max_buffer_size: int = CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE):
    """
    Handle large stream chunks with a buffer that processes chunks as they arrive,
    without storing entire lines in memory.
    
    :param stream: The stream reader to handle.
    :param max_buffer_size: The maximum size of each chunk to yield.
    :return: An async generator yielding chunks of the stream.
    """
    # Initialize buffer with first chunk to start processing
    first_chunk = await stream.readany()
    if not first_chunk:
        return
    
    # Start processing from the beginning of the first chunk
    current_line = first_chunk
    
    async for chunk in stream:
        if not chunk:
            continue
            
        # Combine current_line with new chunk
        data = current_line + chunk
        
        # Process complete lines
        lines = data.split(b'\n')
        
        for line in lines[:-1]:
            # If line exceeds max size, break it into chunks
            if len(line) > max_buffer_size:
                for i in range(0, len(line), max_buffer_size):
                    yield line[i:i + max_buffer_size]
            else:
                yield line
                
        # Handle last partial line
        if lines:
            current_line = lines[-1]
            if len(current_line) > max_buffer_size:
                # If the partial line is too large, break it into chunks
                for i in range(0, len(current_line), max_buffer_size):
                    yield current_line[i:i + max_buffer_size]
            else:
                current_line = lines[-1]  # Keep partial line for next iteration
                
    # Process remaining data in current_line
    if current_line:
        if len(current_line) > max_buffer_size:
            for i in range(0, len(current_line), max_buffer_size):
                yield current_line[i:i + max_buffer_size]
        else:
            yield current_line

async def handle_large_stream_chunks(
    stream: aiohttp.StreamReader, 
    max_buffer_size: int = CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE,
    split_large_content: bool = True  # New parameter
):
    """
    Handle stream response chunks with configurable behavior for oversized lines.

    :param stream: The stream reader to handle.
    :param max_buffer_size: The maximum buffer size in bytes.
    :param split_large_content: If True, split large content; if False, skip it.
    :return: An async generator that yields the stream data.
    """

    buffer = b""
    skip_mode = False

    async for data, _ in stream.iter_chunks():
        if not data:
            continue

        if skip_mode and len(buffer) > max_buffer_size:
            buffer = b""

        lines = (buffer + data).split(b"\n")

        for i in range(len(lines) - 1):
            line = lines[i]

            if skip_mode:
                if len(line) <= max_buffer_size:
                    skip_mode = False
                    yield line
                else:
                    yield b"data: {}"
            else:
                if len(line) > max_buffer_size:
                    if split_large_content:
                        # Try to split the content dynamically
                        try:
                            line_str = line.decode('utf-8', 'replace')
                            if line_str.startswith('data:'):
                                data_str = line_str[len('data:'):].strip()
                                data_obj = json.loads(data_str)

                                # Split large content field
                                choices = data_obj.get('choices', [])
                                if choices and 'delta' in choices[0]:
                                    content = choices[0]['delta'].get('content', '')
                                    if len(content) > max_buffer_size:
                                        # Emit in chunks
                                        for i in range(0, len(content), max_buffer_size):
                                            chunk = content[i:i + max_buffer_size]
                                            chunked_data = {
                                                **data_obj,
                                                'choices': [{
                                                    **choices[0],
                                                    'delta': {**choices[0]['delta'], 'content': chunk}
                                                }]
                                            }
                                            yield f"data: {json.dumps(chunked_data)}\n".encode('utf-8')
                                        continue
                        except Exception as e:
                            log.warning(f"Failed to split large content, skipping: {e}")

                    # Fallback to skip mode
                    skip_mode = True
                    yield b"data: {}"
                    log.info(f"Skip mode triggered, line size: {len(line)}")
                else:
                    yield line

        buffer = lines[-1]

        if not skip_mode and len(buffer) > max_buffer_size:
            skip_mode = True
            log.info(f"Skip mode triggered, buffer size: {len(buffer)}")
            buffer = b""

    if buffer and not skip_mode:
        yield buffer

CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE = int(os.getenv("CHAT_STREAM_RESPONSE_CHUNK_MAX_BUFFER_SIZE", 16384))
CHAT_STREAM_SPLIT_LARGE_CONTENT = os.getenv("CHAT_STREAM_SPLIT_LARGE_CONTENT", "true").lower() == "true"