Generation

Image, audio, and video generation helpers plus response routing utilities.

image_gen

edit_image(prompt, image_path, provider='openai', model=None, height=1024, width=1024, save_path=None, api_key=None)

Convenience function to edit an existing image.

Parameters:
  • prompt (str) –

    The editing instructions.

  • image_path (str) –

    Path to the image to edit.

  • provider (str, default: 'openai' ) –

    The provider to use for editing.

  • model (str, default: None ) –

    The model to use.

  • height (int, default: 1024 ) –

    The height of the output image.

  • width (int, default: 1024 ) –

    The width of the output image.

  • save_path (str, default: None ) –

    Path to save the edited image.

  • api_key (str, default: None ) –

    API key for the provider.

Returns:
  • PIL.Image: The edited image.

Source code in npcpy/gen/image_gen.py
def edit_image(
    prompt: str,
    image_path: str,
    provider: str = "openai",
    model: str = None,
    height: int = 1024,
    width: int = 1024,
    save_path: Optional[str] = None,
    api_key: Optional[str] = None,
):
    """
    Convenience function to edit an existing image.

    Args:
        prompt (str): The editing instructions.
        image_path (str): Path to the image to edit.
        provider (str): The provider to use for editing.
        model (str): The model to use.
        height (int): The height of the output image.
        width (int): The width of the output image.
        save_path (str): Path to save the edited image.
        api_key (str): API key for the provider.

    Returns:
        PIL.Image: The edited image.
    """
    if model is None:
        if provider == "openai":
            model = "gpt-image-1"
        elif provider == "gemini":
            model = "gemini-2.5-flash-image"

    image = generate_image(
        prompt=prompt,
        provider=provider,
        model=model,
        attachments=[image_path],
        height=height,
        width=width,
        save_path=save_path,
        api_key=api_key
    )

    return image

gemini_image_gen(prompt, model='gemini-2.5-flash', attachments=None, n_images=1, api_key=None)

Generate or edit images using Google's Gemini API.

Source code in npcpy/gen/image_gen.py
def gemini_image_gen(
    prompt: str,
    model: str = "gemini-2.5-flash",
    attachments: Union[List[Union[str, bytes, Image.Image]], None] = None,
    n_images: int = 1,
    api_key: Optional[str] = None,
):
    """Generate or edit images using Google's Gemini API."""
    from google import genai
    from google.genai import types
    from io import BytesIO
    import requests

    if api_key is None:
        api_key = os.environ.get('GEMINI_API_KEY')

    client = genai.Client(api_key=api_key)
    collected_images = []

    if attachments is not None:
        processed_contents = [prompt]

        for attachment in attachments:
            if isinstance(attachment, str):
                if attachment.startswith(('http://', 'https://')):
                    image_bytes = requests.get(attachment).content
                    mime_type = 'image/jpeg'
                    processed_contents.append(
                        types.Part.from_bytes(
                            data=image_bytes,
                            mime_type=mime_type
                        )
                    )
                else:
                    with open(attachment, 'rb') as f:
                        image_bytes = f.read()

                    if attachment.lower().endswith('.png'):
                        mime_type = 'image/png'
                    elif attachment.lower().endswith('.jpg') or attachment.lower().endswith('.jpeg'):
                        mime_type = 'image/jpeg'
                    elif attachment.lower().endswith('.webp'):
                        mime_type = 'image/webp'
                    else:
                        mime_type = 'image/jpeg'

                    processed_contents.append(
                        types.Part.from_bytes(
                            data=image_bytes,
                            mime_type=mime_type
                        )
                    )
            elif isinstance(attachment, bytes):
                processed_contents.append(
                    types.Part.from_bytes(
                        data=attachment,
                        mime_type='image/jpeg'
                    )
                )
            elif isinstance(attachment, Image.Image):
                img_byte_arr = BytesIO()
                attachment.save(img_byte_arr, format='PNG')
                img_byte_arr.seek(0)
                processed_contents.append(
                    types.Part.from_bytes(
                        data=img_byte_arr.getvalue(),
                        mime_type='image/png'
                    )
                )

        response = client.models.generate_content(
            model=model,
            contents=processed_contents,
            config=types.GenerateContentConfig(
                response_modalities=["IMAGE", "TEXT"],
            ),
        )

        if hasattr(response, 'candidates') and response.candidates:
            for candidate in response.candidates:
                if candidate.content and candidate.content.parts:
                    for part in candidate.content.parts:
                        if hasattr(part, 'inline_data') and part.inline_data:
                            image_data = part.inline_data.data
                            collected_images.append(Image.open(BytesIO(image_data)))

        if not collected_images and hasattr(response, 'text'):
            print(f"Gemini response text: {response.text}")

        return collected_images
    else:
        if 'imagen' in model:
            response = client.models.generate_images(
                model=model,
                prompt=prompt,
                config=types.GenerateImagesConfig(
                    number_of_images=n_images,
                )
            )
            for generated_image in response.generated_images:
                collected_images.append(Image.open(BytesIO(generated_image.image.image_bytes)))
            return collected_images

        elif 'flash-image' in model or 'image-preview' in model or '2.5-flash' in model:
            response = client.models.generate_content(
                model=model,
                contents=[prompt],
                config=types.GenerateContentConfig(
                    response_modalities=["IMAGE", "TEXT"],
                ),
            )

            if hasattr(response, 'candidates') and response.candidates:
                for candidate in response.candidates:
                    if candidate.content and candidate.content.parts:
                        for part in candidate.content.parts:
                            if hasattr(part, 'inline_data') and part.inline_data:
                                image_data = part.inline_data.data
                                collected_images.append(Image.open(BytesIO(image_data)))

            if not collected_images and hasattr(response, 'text'):
                print(f"Gemini response text: {response.text}")

            return collected_images

        else:
            raise ValueError(f"Unsupported Gemini image model or API usage for new generation: '{model}'")

generate_image(prompt, model, provider, height=1024, width=1024, n_images=1, api_key=None, api_url=None, attachments=None, save_path=None, custom_model_path=None)

Unified function to generate or edit images using various providers.

Parameters:
  • prompt (str) –

    The prompt for generating/editing the image.

  • model (str) –

    The model to use.

  • provider (str) –

    The provider to use ('openai', 'diffusers', 'gemini', 'ollama').

  • height (int, default: 1024 ) –

    The height of the output image.

  • width (int, default: 1024 ) –

    The width of the output image.

  • n_images (int, default: 1 ) –

    Number of images to generate.

  • api_key (str, default: None ) –

    API key for the provider.

  • api_url (str, default: None ) –

    API URL for the provider.

  • attachments (list, default: None ) –

    List of images for editing. Can be file paths, bytes, or PIL Images.

  • save_path (str, default: None ) –

    Path to save the generated image.

  • custom_model_path (str, default: None ) –

    Path to a locally fine-tuned Diffusers model.

Returns:
  • List[PIL.Image.Image]: A list of generated PIL Image objects.

Source code in npcpy/gen/image_gen.py
def generate_image(
    prompt: str,
    model: str ,
    provider: str ,
    height: int = 1024,
    width: int = 1024,
    n_images: int = 1,
    api_key: Optional[str] = None,
    api_url: Optional[str] = None,
    attachments: Union[List[Union[str, bytes, Image.Image]], None] = None,
    save_path: Optional[str] = None,
    custom_model_path: Optional[str] = None,

):
    """
    Unified function to generate or edit images using various providers.

    Args:
        prompt (str): The prompt for generating/editing the image.
        model (str): The model to use.
        provider (str): The provider to use ('openai', 'diffusers', 'gemini', 'ollama').
        height (int): The height of the output image.
        width (int): The width of the output image.
        n_images (int): Number of images to generate.
        api_key (str): API key for the provider.
        api_url (str): API URL for the provider.
        attachments (list): List of images for editing. Can be file paths, bytes, or PIL Images.
        save_path (str): Path to save the generated image.
        custom_model_path (str): Path to a locally fine-tuned Diffusers model.

    Returns:
        List[PIL.Image.Image]: A list of generated PIL Image objects.
    """
    from urllib.request import urlopen
    import os

    if model is None and custom_model_path is None:
        if provider == "openai":
            model = "dall-e-2"
        elif provider == "diffusers":
            model = "runwayml/stable-diffusion-v1-5"
        elif provider == "gemini":
            model = "gemini-2.5-flash-image"
        elif provider == "ollama":
            model = "x/z-image-turbo"

    all_generated_pil_images = []

    if provider == "diffusers":
        if custom_model_path and os.path.isdir(custom_model_path):
            print(f"🌋 Using custom Diffusers model from path: {custom_model_path}")
            model_to_use = custom_model_path
        else:
            model_to_use = model
            print(f"🌋 Using standard Diffusers model: {model_to_use}")

        for _ in range(n_images):
            try:
                image = generate_image_diffusers(
                    prompt=prompt, 
                    model=model_to_use,
                    height=height, 
                    width=width
                )
                all_generated_pil_images.append(image)
            except MemoryError as e:
                raise e
            except Exception as e:
                raise e

    elif provider == "openai":
        images = openai_image_gen(
            prompt=prompt,
            model=model,
            attachments=attachments,
            height=height,
            width=width,
            n_images=n_images,
            api_key=api_key

        )
        all_generated_pil_images.extend(images)

    elif provider == "gemini":
        images = gemini_image_gen(
            prompt=prompt,
            model=model,
            attachments=attachments,
            n_images=n_images,
            api_key=api_key
        )
        all_generated_pil_images.extend(images)

    elif provider == "ollama":
        images = ollama_image_gen(
            prompt=prompt,
            model=model,
            height=height,
            width=width,
            n_images=n_images,
            api_url=api_url
        )
        all_generated_pil_images.extend(images)

    else:
        valid_sizes = ["256x256", "512x512", "1024x1024", "1024x1792", "1792x1024"]
        size = f"{width}x{height}"

        if attachments is not None:
            raise ValueError("Image editing not supported with litellm provider")

        litellm_model_string = f"{provider}/{model}" if provider and model else model

        image_response = image_generation(
            prompt=prompt,
            model=litellm_model_string,
            n=n_images,
            size=size,
            api_key=api_key,
            api_base=api_url,
        )
        for item_data in image_response.data:
            if hasattr(item_data, 'url') and item_data.url:
                with urlopen(item_data.url) as response:
                    all_generated_pil_images.append(Image.open(response))
            elif hasattr(item_data, 'b64_json') and item_data.b64_json:
                image_bytes = base64.b64decode(item_data.b64_json)
                all_generated_pil_images.append(Image.open(io.BytesIO(image_bytes)))
            else:
                print(f"Warning: litellm ImageResponse item has no URL or b64_json: {item_data}")

    if save_path:
        for i, img_item in enumerate(all_generated_pil_images):
            temp_save_path = f"{os.path.splitext(save_path)[0]}_{i}{os.path.splitext(save_path)[1]}"
            if isinstance(img_item, Image.Image):
                img_item.save(temp_save_path)
            else:
                print(f"Warning: Attempting to save non-PIL image item: {type(img_item)}. Skipping save for this item.")

    return all_generated_pil_images

generate_image_diffusers(prompt, model='runwayml/stable-diffusion-v1-5', device='cpu', height=512, width=512)

Generate an image using the Stable Diffusion API with memory optimization.

Source code in npcpy/gen/image_gen.py
def generate_image_diffusers(
    prompt: str,
    model: str = "runwayml/stable-diffusion-v1-5",
    device: str = "cpu",
    height: int = 512,
    width: int = 512,
):
    """Generate an image using the Stable Diffusion API with memory optimization."""
    import torch
    import gc
    import os
    from diffusers import DiffusionPipeline, StableDiffusionPipeline

    try:
        torch_dtype = torch.float16 if device != "cpu" and torch.cuda.is_available() else torch.float32

        if os.path.isdir(model):
            print(f"🌋 Loading fine-tuned Diffusers model from local path: {model}")

            checkpoint_path = os.path.join(model, 'model_final.pt')
            if os.path.exists(checkpoint_path):
                print(f"🌋 Found model_final.pt at {checkpoint_path}.")

                checkpoint = torch.load(checkpoint_path, map_location=device, weights_only=True)

                cfg = checkpoint.get('config')
                has_image_size = (isinstance(cfg, dict) and 'image_size' in cfg) or hasattr(cfg, 'image_size')
                if 'config' in checkpoint and has_image_size:
                    print(f"🌋 Detected custom SimpleUNet model, using custom generation")
                    from npcpy.ft.diff import generate_image as custom_generate_image

                    image = custom_generate_image(
                        model_path=checkpoint_path,
                        prompt=prompt,
                        num_samples=1,
                        image_size=height
                    )
                    return image

                else:
                    print(f"🌋 Detected Stable Diffusion UNet checkpoint")
                    base_model_id = "runwayml/stable-diffusion-v1-5"
                    print(f"🌋 Loading base pipeline: {base_model_id}")
                    pipe = StableDiffusionPipeline.from_pretrained(
                        base_model_id,
                        torch_dtype=torch_dtype,
                        use_safetensors=True,
                        variant="fp16" if torch_dtype == torch.float16 else None,
                    )

                    print(f"🌋 Loading custom UNet weights from {checkpoint_path}")

                    if isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:
                        unet_state_dict = checkpoint['model_state_dict']
                        print(f"🌋 Extracted model_state_dict from checkpoint")
                    else:
                        unet_state_dict = checkpoint
                        print(f"🌋 Using checkpoint directly as state_dict")

                    pipe.unet.load_state_dict(unet_state_dict)
                    pipe = pipe.to(device)
                    print(f"🌋 Successfully loaded fine-tuned UNet weights")

            else:
                raise OSError(f"Error: Fine-tuned model directory {model} does not contain 'model_final.pt'")

        else:
            print(f"🌋 Loading standard Diffusers model: {model}")
            if 'Qwen' in model:
                pipe = DiffusionPipeline.from_pretrained(
                    model,
                    torch_dtype=torch_dtype,
                    use_safetensors=True,
                    variant="fp16" if torch_dtype == torch.float16 else None,
                )
            else:
                pipe = DiffusionPipeline.from_pretrained(
                    model,
                    torch_dtype=torch_dtype,
                    use_safetensors=True,
                    variant="fp16" if torch_dtype == torch.float16 else None,
                )

        if hasattr(pipe, 'enable_attention_slicing'):
            pipe.enable_attention_slicing()

        if hasattr(pipe, 'enable_model_cpu_offload') and device != "cpu":
            pipe.enable_model_cpu_offload()
        else:
            pipe = pipe.to(device)

        if hasattr(pipe, 'enable_xformers_memory_efficient_attention'):
            try:
                pipe.enable_xformers_memory_efficient_attention()
            except Exception:
                pass

        with torch.inference_mode():
            result = pipe(prompt, height=height, width=width, num_inference_steps=20)
            image = result.images[0]

        del pipe
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        gc.collect()

        return image

    except Exception as e:
        if 'pipe' in locals():
            del pipe
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        gc.collect()

        if "out of memory" in str(e).lower() or "killed" in str(e).lower():
            print(f"Memory error during image generation: {e}")
            print("Suggestions:")
            print("1. Try using a smaller model like 'CompVis/stable-diffusion-v1-4'")
            print("2. Reduce image dimensions (e.g., height=256, width=256)")
            print("3. Use a cloud service for image generation")
            raise MemoryError(f"Insufficient memory for image generation with model {model}. Try a smaller model or reduce image size.")
        else:
            raise e

ollama_image_gen(prompt, model='x/z-image-turbo', height=512, width=512, n_images=1, api_url=None, seed=None, negative_prompt=None, num_steps=None)

Generate images using Ollama's image generation API.

Works with ollama image gen models like x/z-image-turbo and x/flux2-klein. Uses the /api/generate endpoint with image gen specific options.

Source code in npcpy/gen/image_gen.py
def ollama_image_gen(
    prompt: str,
    model: str = "x/z-image-turbo",
    height: int = 512,
    width: int = 512,
    n_images: int = 1,
    api_url: Optional[str] = None,
    seed: Optional[int] = None,
    negative_prompt: Optional[str] = None,
    num_steps: Optional[int] = None,
):
    """Generate images using Ollama's image generation API.

    Works with ollama image gen models like x/z-image-turbo and x/flux2-klein.
    Uses the /api/generate endpoint with image gen specific options.
    """
    import requests

    if api_url is None:
        api_url = os.environ.get('OLLAMA_API_URL', 'http://localhost:11434')

    endpoint = f"{api_url}/api/generate"

    collected_images = []

    for _ in range(n_images):
        options = {}
        if width:
            options["width"] = width
        if height:
            options["height"] = height
        if seed is not None:
            options["seed"] = seed
        if num_steps is not None:
            options["num_steps"] = num_steps

        payload = {
            "model": model,
            "prompt": prompt,
            "stream": False,
        }
        if options:
            payload["options"] = options
        if negative_prompt:
            payload["negative_prompt"] = negative_prompt

        response = requests.post(endpoint, json=payload)

        if not response.ok:
            try:
                err = response.json()
                err_msg = err.get('error', response.text)
            except Exception:
                err_msg = response.text
            raise RuntimeError(f"Ollama image gen failed ({response.status_code}): {err_msg}\nModel: {model} — make sure it's pulled (`ollama pull {model}`)")

        result = response.json()

        if 'image' in result and result['image']:
            image_bytes = base64.b64decode(result['image'])
            image = Image.open(io.BytesIO(image_bytes))
            collected_images.append(image)
        elif 'images' in result and result['images']:
            for img_b64 in result['images']:
                image_bytes = base64.b64decode(img_b64)
                image = Image.open(io.BytesIO(image_bytes))
                collected_images.append(image)
        else:
            raise ValueError(f"No images returned from Ollama. Response keys: {list(result.keys())}. Make sure '{model}' is an image generation model (e.g. x/z-image-turbo, x/flux2-klein).")

    return collected_images

openai_image_gen(prompt, model='dall-e-2', attachments=None, height=1024, width=1024, n_images=1, api_key=None)

Generate or edit an image using the OpenAI API.

Source code in npcpy/gen/image_gen.py
def openai_image_gen(
    prompt: str,
    model: str = "dall-e-2",
    attachments: Union[List[Union[str, bytes, Image.Image]], None] = None,
    height: int = 1024,
    width: int = 1024,
    n_images: int = 1,
    api_key: Optional[str] = None,
):
    """Generate or edit an image using the OpenAI API."""
    from openai import OpenAI

    client = OpenAI(api_key=api_key) if api_key else OpenAI()

    if height is None:
        height = 1024
    if width is None:
        width = 1024

    size_str = f"{width}x{height}"

    if attachments is not None:
        processed_images = []
        files_to_close = []
        for attachment in attachments:
            if isinstance(attachment, str):
                file_handle = open(attachment, "rb")
                processed_images.append(file_handle)
                files_to_close.append(file_handle)
            elif isinstance(attachment, bytes):
                img_byte_arr = io.BytesIO(attachment)
                img_byte_arr.name = 'image.png'
                processed_images.append(img_byte_arr)
            elif isinstance(attachment, Image.Image):
                img_byte_arr = io.BytesIO()
                attachment.save(img_byte_arr, format='PNG')
                img_byte_arr.seek(0)
                img_byte_arr.name = 'image.png'
                processed_images.append(img_byte_arr)

        try:
            result = client.images.edit(
                model=model,
                image=processed_images[0],
                prompt=prompt,
                n=n_images,
                size=size_str,
            )
        finally:
            for f in files_to_close:
                f.close()
    else:
        result = client.images.generate(
            model=model,
            prompt=prompt,
            n=n_images,
            size=size_str,
        )

    collected_images = []
    for item_data in result.data:
        b64 = getattr(item_data, 'b64_json', None)
        url = getattr(item_data, 'url', None)
        if b64:
            image_bytes = base64.b64decode(b64)
            image = Image.open(io.BytesIO(image_bytes))
        elif url:
            response = requests.get(url)
            response.raise_for_status()
            image = Image.open(io.BytesIO(response.content))
        else:
            raise ValueError(
                f"OpenAI image response had neither b64_json nor url for model={model}: {item_data!r}"
            )
        collected_images.append(image)

    return collected_images

audio_gen

Audio Generation Module for NPC Supports multiple TTS engines including real-time voice APIs.

TTS Engines: - Kokoro: Local neural TTS (default) - Qwen3-TTS: Local high-quality multilingual TTS (0.6B/1.7B) - ElevenLabs: Cloud TTS with streaming - OpenAI: Realtime voice API - Gemini: Live API for real-time voice - gTTS: Google TTS fallback

Usage

from npcpy.gen.audio_gen import text_to_speech

audio = text_to_speech("Hello world", engine="kokoro", voice="af_heart") audio = text_to_speech("Hello world", engine="qwen3", voice="ryan")

For STT, see npcpy.data.audio

_qwen3_model_cache = {} module-attribute

_get_qwen3_model(model_size='1.7B', model_type='custom_voice', device='auto')

Load and cache a Qwen3-TTS model.

Source code in npcpy/gen/audio_gen.py
def _get_qwen3_model(
    model_size: str = "1.7B",
    model_type: str = "custom_voice",
    device: str = "auto",
):
    """Load and cache a Qwen3-TTS model."""
    cache_key = (model_size, model_type, device)
    if cache_key in _qwen3_model_cache:
        return _qwen3_model_cache[cache_key]

    import torch
    from huggingface_hub import snapshot_download

    if device == "auto":
        if torch.cuda.is_available():
            device = "cuda"
        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
            device = "mps"
        else:
            device = "cpu"

    dtype = torch.bfloat16 if device != "cpu" else torch.float32

    size_tag = "0.6B" if "0.6" in model_size else "1.7B"
    type_map = {
        "custom_voice": f"Qwen/Qwen3-TTS-12Hz-{size_tag}-CustomVoice",
        "base": f"Qwen/Qwen3-TTS-12Hz-{size_tag}-Base",
        "voice_design": f"Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign",
    }

    repo_id = type_map.get(model_type, type_map["custom_voice"])

    cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "qwen-tts")
    model_dir = os.path.join(cache_dir, repo_id.split("/")[-1])

    if not os.path.exists(os.path.join(model_dir, "config.json")):
        os.makedirs(cache_dir, exist_ok=True)
        snapshot_download(repo_id=repo_id, local_dir=model_dir)

    try:
        from qwen_tts import Qwen3TTSModel
    except ImportError:
        raise ImportError("qwen_tts package not found. Install from: https://github.com/QwenLM/Qwen3-TTS or pip install qwen-tts")

    model = Qwen3TTSModel.from_pretrained(
        model_dir, device_map=device, dtype=dtype
    )

    _qwen3_model_cache.clear()
    _qwen3_model_cache[cache_key] = model
    return model

_music_one(provider, prompt, model, duration, api_key)

Source code in npcpy/gen/audio_gen.py
def _music_one(provider: str, prompt: str, model: Optional[str], duration: int, api_key: Optional[str]) -> dict:
    p = (provider or "").lower()
    if p in ("local", "musicgen", "transformers", "meta"):
        m = model or "facebook/musicgen-small"
        return {"audio": music_musicgen_local(prompt, duration=duration, model=m),
                "format": "wav", "provider": "musicgen-local", "model": m}
    if p in ("replicate",):
        m = model or "meta/musicgen"
        return {"audio": music_replicate(prompt, duration=duration, model=m, api_key=api_key),
                "format": "wav", "provider": "replicate", "model": m}
    if p in ("elevenlabs", "stability", "11labs"):
        return {"audio": music_elevenlabs_sfx(prompt, duration=duration, api_key=api_key),
                "format": "mp3", "provider": "elevenlabs", "model": "sound-generation"}
    raise ValueError(f"Unknown music provider: {provider!r}")

audio_to_base64(audio_data)

Encode audio to base64 string.

Source code in npcpy/gen/audio_gen.py
def audio_to_base64(audio_data: bytes) -> str:
    """Encode audio to base64 string."""
    return base64.b64encode(audio_data).decode('utf-8')

base64_to_audio(b64_string)

Decode base64 string to audio bytes.

Source code in npcpy/gen/audio_gen.py
def base64_to_audio(b64_string: str) -> bytes:
    """Decode base64 string to audio bytes."""
    return base64.b64decode(b64_string)

gemini_live_connect(api_key=None, model='gemini-2.0-flash-exp', voice='Puck', system_instruction='You are a helpful assistant.') async

Connect to Gemini Live API.

Returns:
  • WebSocket connection

Source code in npcpy/gen/audio_gen.py
async def gemini_live_connect(
    api_key: Optional[str] = None,
    model: str = "gemini-2.0-flash-exp",
    voice: str = "Puck",
    system_instruction: str = "You are a helpful assistant."
):
    """
    Connect to Gemini Live API.

    Returns:
        WebSocket connection
    """
    import websockets

    api_key = api_key or os.environ.get('GOOGLE_API_KEY') or os.environ.get('GEMINI_API_KEY')

    url = f"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={api_key}"

    ws = await websockets.connect(url)

    await ws.send(json.dumps({
        "setup": {
            "model": f"models/{model}",
            "generation_config": {
                "response_modalities": ["AUDIO"],
                "speech_config": {
                    "voice_config": {
                        "prebuilt_voice_config": {"voice_name": voice}
                    }
                }
            },
            "system_instruction": {"parts": [{"text": system_instruction}]}
        }
    }))

    response = await ws.recv()
    data = json.loads(response)
    if "setupComplete" not in data:
        await ws.close()
        raise Exception(f"Gemini Live setup failed: {data}")

    return ws

gemini_live_receive(ws, on_audio=None, on_text=None) async

Receive response from Gemini Live.

Returns:
  • Tuple of (full_audio_bytes, full_text)

Source code in npcpy/gen/audio_gen.py
async def gemini_live_receive(ws, on_audio=None, on_text=None):
    """
    Receive response from Gemini Live.

    Returns:
        Tuple of (full_audio_bytes, full_text)
    """
    audio_chunks = []
    text_chunks = []

    async for message in ws:
        data = json.loads(message)

        if "serverContent" in data:
            content = data["serverContent"]

            if "modelTurn" in content:
                for part in content["modelTurn"].get("parts", []):
                    if "inlineData" in part:
                        audio = base64.b64decode(part["inlineData"].get("data", ""))
                        audio_chunks.append(audio)
                        if on_audio:
                            on_audio(audio)
                    elif "text" in part:
                        text_chunks.append(part["text"])
                        if on_text:
                            on_text(part["text"])

            if content.get("turnComplete"):
                break

    return b''.join(audio_chunks), ''.join(text_chunks)

gemini_live_send_audio(ws, audio_data, mime_type='audio/pcm') async

Send audio to Gemini Live.

Source code in npcpy/gen/audio_gen.py
async def gemini_live_send_audio(ws, audio_data: bytes, mime_type: str = "audio/pcm"):
    """Send audio to Gemini Live."""
    await ws.send(json.dumps({
        "realtime_input": {
            "media_chunks": [{
                "data": base64.b64encode(audio_data).decode(),
                "mime_type": mime_type
            }]
        }
    }))

gemini_live_send_text(ws, text) async

Send text message to Gemini Live.

Source code in npcpy/gen/audio_gen.py
async def gemini_live_send_text(ws, text: str):
    """Send text message to Gemini Live."""
    await ws.send(json.dumps({
        "client_content": {
            "turns": [{"role": "user", "parts": [{"text": text}]}],
            "turn_complete": True
        }
    }))

generate_music(prompt, provider='replicate', model=None, duration=10, api_key=None)

Unified music-generation entry point with automatic provider fallback.

Tries the requested provider first; on failure, walks the other providers that have credentials available so the user always gets audio back.

Source code in npcpy/gen/audio_gen.py
def generate_music(
    prompt: str,
    provider: str = "replicate",
    model: Optional[str] = None,
    duration: int = 10,
    api_key: Optional[str] = None,
) -> dict:
    """Unified music-generation entry point with automatic provider fallback.

    Tries the requested provider first; on failure, walks the other providers
    that have credentials available so the user always gets audio back.
    """
    requested = (provider or "local").lower()
    order = [requested]
    for cand in ("local", "replicate", "elevenlabs"):
        if cand not in order:
            order.append(cand)

    errors: list[str] = []
    for prov in order:
        # Skip cloud providers that have no credentials configured.
        if prov == "replicate" and not (api_key or os.environ.get("REPLICATE_API_TOKEN") or os.environ.get("REPLICATE_API_KEY")):
            errors.append(f"{prov}: no REPLICATE_API_TOKEN")
            continue
        if prov in ("elevenlabs", "11labs", "stability") and not (api_key or os.environ.get("ELEVENLABS_API_KEY")):
            errors.append(f"{prov}: no ELEVENLABS_API_KEY")
            continue
        try:
            return _music_one(prov, prompt, model if prov == requested else None, duration, api_key)
        except Exception as e:
            errors.append(f"{prov}: {e}")
            continue

    raise RuntimeError("All music providers failed:\n  - " + "\n  - ".join(errors))

get_available_engines()

Get info about available TTS engines.

Source code in npcpy/gen/audio_gen.py
def get_available_engines() -> dict:
    """Get info about available TTS engines."""
    engines = {
        "kokoro": {
            "name": "Kokoro",
            "type": "local",
            "available": False,
            "description": "Local neural TTS (82M params)",
            "install": "pip install kokoro soundfile"
        },
        "qwen3": {
            "name": "Qwen3-TTS",
            "type": "local",
            "available": False,
            "description": "Local high-quality multilingual TTS (0.6B/1.7B)",
            "install": "pip install qwen-tts torch torchaudio transformers"
        },
        "elevenlabs": {
            "name": "ElevenLabs",
            "type": "cloud",
            "available": False,
            "description": "High-quality cloud TTS",
            "requires": "ELEVENLABS_API_KEY"
        },
        "openai": {
            "name": "OpenAI Realtime",
            "type": "cloud",
            "available": False,
            "description": "OpenAI real-time voice API",
            "requires": "OPENAI_API_KEY"
        },
        "gemini": {
            "name": "Gemini Live",
            "type": "cloud",
            "available": False,
            "description": "Google Gemini real-time voice",
            "requires": "GOOGLE_API_KEY or GEMINI_API_KEY"
        },
        "gtts": {
            "name": "Google TTS",
            "type": "cloud",
            "available": False,
            "description": "Free Google TTS (fallback)"
        }
    }

    try:
        from kokoro import KPipeline
        engines["kokoro"]["available"] = True
    except ImportError:
        pass

    try:
        from qwen_tts import Qwen3TTSModel
        engines["qwen3"]["available"] = True
    except ImportError:
        pass

    if os.environ.get('ELEVENLABS_API_KEY'):
        engines["elevenlabs"]["available"] = True

    if os.environ.get('OPENAI_API_KEY'):
        engines["openai"]["available"] = True

    if os.environ.get('GOOGLE_API_KEY') or os.environ.get('GEMINI_API_KEY'):
        engines["gemini"]["available"] = True

    try:
        from gtts import gTTS
        engines["gtts"]["available"] = True
    except ImportError:
        pass

    return engines

get_available_voices(engine='kokoro')

Get available voices for an engine.

Source code in npcpy/gen/audio_gen.py
def get_available_voices(engine: str = "kokoro") -> list:
    """Get available voices for an engine."""
    engine = engine.lower()

    if engine == "kokoro":
        return get_kokoro_voices()
    elif engine in ("qwen3", "qwen3-tts", "qwen"):
        return get_qwen3_voices()
    elif engine == "elevenlabs":
        return get_elevenlabs_voices()
    elif engine == "openai":
        return get_openai_voices()
    elif engine == "gemini":
        return get_gemini_voices()
    elif engine == "gtts":
        return get_gtts_voices()
    else:
        return []

get_elevenlabs_voices(api_key=None)

Get available ElevenLabs voices.

Source code in npcpy/gen/audio_gen.py
def get_elevenlabs_voices(api_key: Optional[str] = None) -> list:
    """Get available ElevenLabs voices."""
    if api_key is None:
        api_key = os.environ.get('ELEVENLABS_API_KEY')

    if not api_key:
        return []

    try:
        from elevenlabs.client import ElevenLabs
        client = ElevenLabs(api_key=api_key)
        voices = client.voices.get_all()
        return [{"id": v.voice_id, "name": v.name} for v in voices.voices]
    except Exception:
        return []

get_gemini_voices()

Get available Gemini Live voices.

Source code in npcpy/gen/audio_gen.py
def get_gemini_voices() -> list:
    """Get available Gemini Live voices."""
    return [
        {"id": "Puck", "name": "Puck"},
        {"id": "Charon", "name": "Charon"},
        {"id": "Kore", "name": "Kore"},
        {"id": "Fenrir", "name": "Fenrir"},
        {"id": "Aoede", "name": "Aoede"},
    ]

get_gtts_voices()

Get available gTTS languages.

Source code in npcpy/gen/audio_gen.py
def get_gtts_voices() -> list:
    """Get available gTTS languages."""
    return [
        {"id": "en", "name": "English"},
        {"id": "es", "name": "Spanish"},
        {"id": "fr", "name": "French"},
        {"id": "de", "name": "German"},
        {"id": "it", "name": "Italian"},
        {"id": "pt", "name": "Portuguese"},
        {"id": "ja", "name": "Japanese"},
        {"id": "ko", "name": "Korean"},
        {"id": "zh-CN", "name": "Chinese"},
    ]

get_kokoro_voices()

Get available Kokoro voices.

Source code in npcpy/gen/audio_gen.py
def get_kokoro_voices() -> list:
    """Get available Kokoro voices."""
    return [
        {"id": "af_heart", "name": "Heart", "gender": "female", "lang": "a"},
        {"id": "af_bella", "name": "Bella", "gender": "female", "lang": "a"},
        {"id": "af_sarah", "name": "Sarah", "gender": "female", "lang": "a"},
        {"id": "af_nicole", "name": "Nicole", "gender": "female", "lang": "a"},
        {"id": "af_sky", "name": "Sky", "gender": "female", "lang": "a"},
        {"id": "am_adam", "name": "Adam", "gender": "male", "lang": "a"},
        {"id": "am_michael", "name": "Michael", "gender": "male", "lang": "a"},
        {"id": "bf_emma", "name": "Emma", "gender": "female", "lang": "b"},
        {"id": "bf_isabella", "name": "Isabella", "gender": "female", "lang": "b"},
        {"id": "bm_george", "name": "George", "gender": "male", "lang": "b"},
        {"id": "bm_lewis", "name": "Lewis", "gender": "male", "lang": "b"},
    ]

get_openai_voices()

Get available OpenAI Realtime voices.

Source code in npcpy/gen/audio_gen.py
def get_openai_voices() -> list:
    """Get available OpenAI Realtime voices."""
    return [
        {"id": "alloy", "name": "Alloy"},
        {"id": "echo", "name": "Echo"},
        {"id": "shimmer", "name": "Shimmer"},
        {"id": "ash", "name": "Ash"},
        {"id": "ballad", "name": "Ballad"},
        {"id": "coral", "name": "Coral"},
        {"id": "sage", "name": "Sage"},
        {"id": "verse", "name": "Verse"},
    ]

get_qwen3_voices()

Get available Qwen3-TTS preset voices.

Source code in npcpy/gen/audio_gen.py
def get_qwen3_voices() -> list:
    """Get available Qwen3-TTS preset voices."""
    return [
        {"id": "aiden", "name": "Aiden", "gender": "male"},
        {"id": "dylan", "name": "Dylan", "gender": "male"},
        {"id": "eric", "name": "Eric", "gender": "male"},
        {"id": "ryan", "name": "Ryan", "gender": "male"},
        {"id": "serena", "name": "Serena", "gender": "female"},
        {"id": "vivian", "name": "Vivian", "gender": "female"},
        {"id": "sohee", "name": "Sohee", "gender": "female"},
        {"id": "ono_anna", "name": "Ono Anna", "gender": "female"},
        {"id": "uncle_fu", "name": "Uncle Fu", "gender": "male"},
    ]

music_elevenlabs_sfx(prompt, duration=10.0, api_key=None)

Generate a short sound effect / music clip via ElevenLabs sound-generation API.

Source code in npcpy/gen/audio_gen.py
def music_elevenlabs_sfx(
    prompt: str,
    duration: float = 10.0,
    api_key: Optional[str] = None,
) -> bytes:
    """Generate a short sound effect / music clip via ElevenLabs sound-generation API."""
    import requests

    api_key = api_key or os.environ.get("ELEVENLABS_API_KEY")
    if not api_key:
        raise RuntimeError("ELEVENLABS_API_KEY env var required for ElevenLabs sound generation")

    # ElevenLabs caps sound-generation at 22 seconds.
    duration = max(0.5, min(22.0, float(duration)))

    r = requests.post(
        "https://api.elevenlabs.io/v1/sound-generation",
        headers={"xi-api-key": api_key, "Content-Type": "application/json"},
        json={
            "text": prompt,
            "duration_seconds": duration,
            "prompt_influence": 0.3,
        },
        timeout=180,
    )
    r.raise_for_status()
    return r.content  # MP3 bytes

music_musicgen_local(prompt, duration=10, model='facebook/musicgen-small')

Generate music locally with Meta MusicGen via transformers.

Returns WAV bytes (MusicGen outputs at 32 kHz).

Source code in npcpy/gen/audio_gen.py
def music_musicgen_local(
    prompt: str,
    duration: int = 10,
    model: str = "facebook/musicgen-small",
) -> bytes:
    """Generate music locally with Meta MusicGen via transformers.

    Returns WAV bytes (MusicGen outputs at 32 kHz).
    """
    import numpy as np
    import scipy.io.wavfile as wavfile
    import torch
    # Use concrete classes to sidestep AutoProcessor's auto-discovery path.
    from transformers import MusicgenProcessor, MusicgenForConditionalGeneration

    processor = MusicgenProcessor.from_pretrained(model)
    mg = MusicgenForConditionalGeneration.from_pretrained(model)

    # MusicGen is numerically unstable on MPS (produces NaN/inf probs during
    # sampling) — stick to CUDA or CPU.
    if torch.cuda.is_available():
        device = "cuda"
    else:
        device = "cpu"
    mg = mg.to(device)
    mg.eval()

    inputs = processor(text=[prompt], padding=True, return_tensors="pt").to(device)

    # MusicGen uses ~50 tokens/sec at 32 kHz
    max_new_tokens = int(duration * 50)

    with torch.no_grad():
        audio_values = mg.generate(**inputs, max_new_tokens=max_new_tokens, do_sample=True, guidance_scale=3.0)

    sample_rate = mg.config.audio_encoder.sampling_rate
    arr = audio_values[0, 0].detach().cpu().numpy()
    # normalize to int16
    arr = np.clip(arr, -1.0, 1.0)
    pcm = (arr * 32767).astype(np.int16)

    buf = io.BytesIO()
    wavfile.write(buf, sample_rate, pcm)
    return buf.getvalue()

music_replicate(prompt, duration=10, model='meta/musicgen', version=None, api_key=None)

Generate music via Replicate. Works with musicgen, stable-audio, riffusion.

Source code in npcpy/gen/audio_gen.py
def music_replicate(
    prompt: str,
    duration: int = 10,
    model: str = "meta/musicgen",
    version: Optional[str] = None,
    api_key: Optional[str] = None,
) -> bytes:
    """Generate music via Replicate. Works with musicgen, stable-audio, riffusion."""
    import requests

    api_key = api_key or os.environ.get("REPLICATE_API_TOKEN") or os.environ.get("REPLICATE_API_KEY")
    if not api_key:
        raise RuntimeError("REPLICATE_API_TOKEN env var required for Replicate music generation")

    # Resolve model → latest version automatically if not given.
    if not version:
        owner_model = model
        r = requests.get(
            f"https://api.replicate.com/v1/models/{owner_model}",
            headers={"Authorization": f"Token {api_key}"},
            timeout=30,
        )
        r.raise_for_status()
        version = r.json()["latest_version"]["id"]

    inputs: dict = {"prompt": prompt, "duration": duration}
    if "stable-audio" in model:
        inputs = {"prompt": prompt, "seconds_total": duration}
    elif "riffusion" in model:
        inputs = {"prompt_a": prompt}

    create = requests.post(
        "https://api.replicate.com/v1/predictions",
        headers={"Authorization": f"Token {api_key}", "Content-Type": "application/json"},
        json={"version": version, "input": inputs},
        timeout=30,
    )
    create.raise_for_status()
    pred = create.json()
    poll_url = pred["urls"]["get"]

    import time
    for _ in range(180):  # up to ~6 minutes
        p = requests.get(poll_url, headers={"Authorization": f"Token {api_key}"}, timeout=30).json()
        status = p.get("status")
        if status == "succeeded":
            out = p.get("output")
            audio_url = out[0] if isinstance(out, list) else out
            return requests.get(audio_url, timeout=120).content
        if status in ("failed", "canceled"):
            raise RuntimeError(f"Replicate prediction {status}: {p.get('error')}")
        time.sleep(2)
    raise TimeoutError("Replicate music generation timed out")

openai_realtime_connect(api_key=None, model='gpt-4o-realtime-preview-2024-12-17', voice='alloy', instructions='You are a helpful assistant.') async

Connect to OpenAI Realtime API.

Returns:
  • WebSocket connection

Source code in npcpy/gen/audio_gen.py
async def openai_realtime_connect(
    api_key: Optional[str] = None,
    model: str = "gpt-4o-realtime-preview-2024-12-17",
    voice: str = "alloy",
    instructions: str = "You are a helpful assistant."
):
    """
    Connect to OpenAI Realtime API.

    Returns:
        WebSocket connection
    """
    import websockets

    api_key = api_key or os.environ.get('OPENAI_API_KEY')

    url = f"wss://api.openai.com/v1/realtime?model={model}"
    headers = {
        "Authorization": f"Bearer {api_key}",
        "OpenAI-Beta": "realtime=v1"
    }

    ws = await websockets.connect(url, extra_headers=headers)

    await ws.send(json.dumps({
        "type": "session.update",
        "session": {
            "modalities": ["text", "audio"],
            "instructions": instructions,
            "voice": voice,
            "input_audio_format": "pcm16",
            "output_audio_format": "pcm16",
            "input_audio_transcription": {"model": "whisper-1"},
            "turn_detection": {
                "type": "server_vad",
                "threshold": 0.5,
                "prefix_padding_ms": 300,
                "silence_duration_ms": 500
            }
        }
    }))

    while True:
        msg = await ws.recv()
        event = json.loads(msg)
        if event.get("type") == "session.created":
            break
        elif event.get("type") == "error":
            await ws.close()
            raise Exception(f"OpenAI Realtime error: {event}")

    return ws

openai_realtime_receive(ws, on_audio=None, on_text=None) async

Receive response from OpenAI Realtime.

Parameters:
  • ws

    WebSocket connection

  • on_audio

    Callback for audio chunks (bytes)

  • on_text

    Callback for text chunks (str)

Returns:
  • Tuple of (full_audio_bytes, full_text)

Source code in npcpy/gen/audio_gen.py
async def openai_realtime_receive(ws, on_audio=None, on_text=None):
    """
    Receive response from OpenAI Realtime.

    Args:
        ws: WebSocket connection
        on_audio: Callback for audio chunks (bytes)
        on_text: Callback for text chunks (str)

    Returns:
        Tuple of (full_audio_bytes, full_text)
    """
    audio_chunks = []
    text_chunks = []

    async for message in ws:
        event = json.loads(message)
        event_type = event.get("type", "")

        if event_type == "response.audio.delta":
            audio = base64.b64decode(event.get("delta", ""))
            audio_chunks.append(audio)
            if on_audio:
                on_audio(audio)

        elif event_type == "response.text.delta":
            text = event.get("delta", "")
            text_chunks.append(text)
            if on_text:
                on_text(text)

        elif event_type == "response.done":
            break

    return b''.join(audio_chunks), ''.join(text_chunks)

openai_realtime_send_audio(ws, audio_data) async

Send audio to OpenAI Realtime (PCM16, 24kHz, mono).

Source code in npcpy/gen/audio_gen.py
async def openai_realtime_send_audio(ws, audio_data: bytes):
    """Send audio to OpenAI Realtime (PCM16, 24kHz, mono)."""
    await ws.send(json.dumps({
        "type": "input_audio_buffer.append",
        "audio": base64.b64encode(audio_data).decode()
    }))

openai_realtime_send_text(ws, text) async

Send text message to OpenAI Realtime.

Source code in npcpy/gen/audio_gen.py
async def openai_realtime_send_text(ws, text: str):
    """Send text message to OpenAI Realtime."""
    await ws.send(json.dumps({
        "type": "conversation.item.create",
        "item": {
            "type": "message",
            "role": "user",
            "content": [{"type": "input_text", "text": text}]
        }
    }))
    await ws.send(json.dumps({"type": "response.create"}))

pcm16_to_wav(pcm_data, sample_rate=24000, channels=1)

Convert raw PCM16 audio to WAV format.

Source code in npcpy/gen/audio_gen.py
def pcm16_to_wav(pcm_data: bytes, sample_rate: int = 24000, channels: int = 1) -> bytes:
    """Convert raw PCM16 audio to WAV format."""
    import struct

    wav_buffer = io.BytesIO()
    wav_buffer.write(b'RIFF')
    wav_buffer.write(struct.pack('<I', 36 + len(pcm_data)))
    wav_buffer.write(b'WAVE')
    wav_buffer.write(b'fmt ')
    wav_buffer.write(struct.pack('<I', 16))
    wav_buffer.write(struct.pack('<H', 1))
    wav_buffer.write(struct.pack('<H', channels))
    wav_buffer.write(struct.pack('<I', sample_rate))
    wav_buffer.write(struct.pack('<I', sample_rate * channels * 2))
    wav_buffer.write(struct.pack('<H', channels * 2))
    wav_buffer.write(struct.pack('<H', 16))
    wav_buffer.write(b'data')
    wav_buffer.write(struct.pack('<I', len(pcm_data)))
    wav_buffer.write(pcm_data)

    wav_buffer.seek(0)
    return wav_buffer.read()

text_to_speech(text, engine='kokoro', voice=None, **kwargs)

Unified TTS interface.

Parameters:
  • text (str) –

    Text to synthesize

  • engine (str, default: 'kokoro' ) –

    TTS engine (kokoro, qwen3, elevenlabs, openai, gemini, gtts)

  • voice (Optional[str], default: None ) –

    Voice ID (engine-specific)

  • **kwargs

    Engine-specific options

Returns:
  • bytes

    Audio bytes (format depends on engine)

Source code in npcpy/gen/audio_gen.py
def text_to_speech(
    text: str,
    engine: str = "kokoro",
    voice: Optional[str] = None,
    **kwargs
) -> bytes:
    """
    Unified TTS interface.

    Args:
        text: Text to synthesize
        engine: TTS engine (kokoro, qwen3, elevenlabs, openai, gemini, gtts)
        voice: Voice ID (engine-specific)
        **kwargs: Engine-specific options

    Returns:
        Audio bytes (format depends on engine)
    """
    engine = engine.lower()

    if engine == "kokoro":
        voice = voice or "af_heart"
        voices = {v["id"]: v for v in get_kokoro_voices()}
        lang_code = voices.get(voice, {}).get("lang", "a")
        return tts_kokoro(text, voice=voice, lang_code=lang_code, **kwargs)

    elif engine in ("qwen3", "qwen3-tts", "qwen"):
        voice = voice or "ryan"
        return tts_qwen3(text, voice=voice, **kwargs)

    elif engine == "elevenlabs":
        voice = voice or "JBFqnCBsd6RMkjVDRZzb"
        return tts_elevenlabs(text, voice_id=voice, **kwargs)

    elif engine == "openai":
        voice = voice or "alloy"
        return asyncio.run(tts_openai_realtime(text, voice=voice, **kwargs))

    elif engine == "gemini":
        voice = voice or "Puck"
        return asyncio.run(tts_gemini_live(text, voice=voice, **kwargs))

    elif engine == "gtts":
        lang = voice if voice and len(voice) <= 5 else "en"
        return tts_gtts(text, lang=lang)

    else:
        raise ValueError(f"Unknown TTS engine: {engine}")

tts_elevenlabs(text, api_key=None, voice_id='JBFqnCBsd6RMkjVDRZzb', model_id='eleven_multilingual_v2', output_format='mp3_44100_128')

Generate speech using ElevenLabs API.

Returns:
  • bytes

    MP3 audio bytes

Source code in npcpy/gen/audio_gen.py
def tts_elevenlabs(
    text: str,
    api_key: Optional[str] = None,
    voice_id: str = 'JBFqnCBsd6RMkjVDRZzb',
    model_id: str = 'eleven_multilingual_v2',
    output_format: str = 'mp3_44100_128'
) -> bytes:
    """
    Generate speech using ElevenLabs API.

    Returns:
        MP3 audio bytes
    """
    if api_key is None:
        api_key = os.environ.get('ELEVENLABS_API_KEY')

    if not api_key:
        raise ValueError("ELEVENLABS_API_KEY not set")

    from elevenlabs.client import ElevenLabs

    client = ElevenLabs(api_key=api_key)

    audio_generator = client.text_to_speech.convert(
        text=text,
        voice_id=voice_id,
        model_id=model_id,
        output_format=output_format
    )

    return b''.join(chunk for chunk in audio_generator)

tts_elevenlabs_stream(text, api_key=None, voice_id='JBFqnCBsd6RMkjVDRZzb', model_id='eleven_turbo_v2_5', on_chunk=None) async

Stream TTS via ElevenLabs WebSocket for lowest latency.

Parameters:
  • text (str) –

    Text to synthesize

  • api_key (Optional[str], default: None ) –

    ElevenLabs API key

  • voice_id (str, default: 'JBFqnCBsd6RMkjVDRZzb' ) –

    Voice to use

  • model_id (str, default: 'eleven_turbo_v2_5' ) –

    Model (eleven_turbo_v2_5 for fastest)

  • on_chunk (Optional[Callable[[bytes], None]], default: None ) –

    Callback for each audio chunk

Returns:
  • bytes

    Complete audio bytes

Source code in npcpy/gen/audio_gen.py
async def tts_elevenlabs_stream(
    text: str,
    api_key: Optional[str] = None,
    voice_id: str = 'JBFqnCBsd6RMkjVDRZzb',
    model_id: str = 'eleven_turbo_v2_5',
    on_chunk: Optional[Callable[[bytes], None]] = None
) -> bytes:
    """
    Stream TTS via ElevenLabs WebSocket for lowest latency.

    Args:
        text: Text to synthesize
        api_key: ElevenLabs API key
        voice_id: Voice to use
        model_id: Model (eleven_turbo_v2_5 for fastest)
        on_chunk: Callback for each audio chunk

    Returns:
        Complete audio bytes
    """
    import websockets

    if api_key is None:
        api_key = os.environ.get('ELEVENLABS_API_KEY')

    uri = f"wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id={model_id}"

    all_audio = []

    async with websockets.connect(uri) as ws:
        await ws.send(json.dumps({
            "text": " ",
            "voice_settings": {"stability": 0.5, "similarity_boost": 0.75},
            "xi_api_key": api_key
        }))

        await ws.send(json.dumps({"text": text}))
        await ws.send(json.dumps({"text": ""}))

        async for message in ws:
            data = json.loads(message)
            if "audio" in data:
                chunk = base64.b64decode(data["audio"])
                all_audio.append(chunk)
                if on_chunk:
                    on_chunk(chunk)
            if data.get("isFinal"):
                break

    return b''.join(all_audio)

tts_gemini_live(text, api_key=None, voice='Puck', on_chunk=None) async

Use Gemini Live API for TTS.

Returns PCM audio.

Source code in npcpy/gen/audio_gen.py
async def tts_gemini_live(
    text: str,
    api_key: Optional[str] = None,
    voice: str = "Puck",
    on_chunk: Optional[Callable[[bytes], None]] = None
) -> bytes:
    """
    Use Gemini Live API for TTS.

    Returns PCM audio.
    """
    ws = await gemini_live_connect(api_key=api_key, voice=voice)
    try:
        await gemini_live_send_text(ws, f"Please repeat exactly: {text}")
        audio, _ = await gemini_live_receive(ws, on_audio=on_chunk)
        return audio
    finally:
        await ws.close()

tts_gtts(text, lang='en')

Generate speech using gTTS.

Returns MP3 audio bytes.

Source code in npcpy/gen/audio_gen.py
def tts_gtts(text: str, lang: str = "en") -> bytes:
    """
    Generate speech using gTTS.

    Returns MP3 audio bytes.
    """
    from gtts import gTTS

    tts = gTTS(text=text, lang=lang)

    mp3_buffer = io.BytesIO()
    tts.write_to_fp(mp3_buffer)
    mp3_buffer.seek(0)
    return mp3_buffer.read()

tts_kokoro(text, voice='af_heart', lang_code='a', speed=1.0)

Generate speech using Kokoro local neural TTS.

Parameters:
  • text (str) –

    Text to synthesize

  • voice (str, default: 'af_heart' ) –

    Voice ID (af_heart, am_adam, bf_emma, etc.)

  • lang_code (str, default: 'a' ) –

    'a' for American, 'b' for British

  • speed (float, default: 1.0 ) –

    Speech speed multiplier

Returns:
  • bytes

    WAV audio bytes

Source code in npcpy/gen/audio_gen.py
def tts_kokoro(
    text: str,
    voice: str = "af_heart",
    lang_code: str = "a",
    speed: float = 1.0
) -> bytes:
    """
    Generate speech using Kokoro local neural TTS.

    Args:
        text: Text to synthesize
        voice: Voice ID (af_heart, am_adam, bf_emma, etc.)
        lang_code: 'a' for American, 'b' for British
        speed: Speech speed multiplier

    Returns:
        WAV audio bytes
    """
    from kokoro import KPipeline
    import soundfile as sf
    import numpy as np

    pipeline = KPipeline(lang_code=lang_code)

    audio_chunks = []
    for _, _, audio in pipeline(text, voice=voice, speed=speed):
        audio_chunks.append(audio)

    if not audio_chunks:
        raise ValueError("No audio generated")

    full_audio = np.concatenate(audio_chunks)

    wav_buffer = io.BytesIO()
    sf.write(wav_buffer, full_audio, 24000, format='WAV')
    wav_buffer.seek(0)
    return wav_buffer.read()

tts_openai_realtime(text, api_key=None, voice='alloy', on_chunk=None) async

Use OpenAI Realtime API for TTS.

Returns PCM16 audio at 24kHz.

Source code in npcpy/gen/audio_gen.py
async def tts_openai_realtime(
    text: str,
    api_key: Optional[str] = None,
    voice: str = "alloy",
    on_chunk: Optional[Callable[[bytes], None]] = None
) -> bytes:
    """
    Use OpenAI Realtime API for TTS.

    Returns PCM16 audio at 24kHz.
    """
    ws = await openai_realtime_connect(api_key=api_key, voice=voice)
    try:
        await openai_realtime_send_text(ws, f"Please repeat exactly: {text}")
        audio, _ = await openai_realtime_receive(ws, on_audio=on_chunk)
        return audio
    finally:
        await ws.close()

tts_qwen3(text, voice='ryan', language='auto', model_size='1.7B', device='auto', speed=1.0, ref_audio=None, ref_text=None, instruct=None)

Generate speech using Qwen3-TTS local model.

Supports three modes based on arguments: - Custom voice (default): Use a preset speaker name - Voice clone: Provide ref_audio (path) to clone a voice - Voice design: Provide instruct (text description) to design a voice

Parameters:
  • text (str) –

    Text to synthesize

  • voice (str, default: 'ryan' ) –

    Speaker name for custom voice mode (aiden, dylan, eric, ono_anna, ryan, serena, sohee, uncle_fu, vivian)

  • language (str, default: 'auto' ) –

    Language (auto, chinese, english, japanese, korean, french, etc.)

  • model_size (str, default: '1.7B' ) –

    '0.6B' or '1.7B'

  • device (str, default: 'auto' ) –

    'auto', 'cuda', 'mps', 'cpu'

  • speed (float, default: 1.0 ) –

    Speech speed (not directly supported, reserved)

  • ref_audio (str, default: None ) –

    Path to reference audio for voice cloning

  • ref_text (str, default: None ) –

    Transcript of reference audio (recommended for cloning)

  • instruct (str, default: None ) –

    Natural language voice description for voice design mode

Returns:
  • bytes

    WAV audio bytes

Source code in npcpy/gen/audio_gen.py
def tts_qwen3(
    text: str,
    voice: str = "ryan",
    language: str = "auto",
    model_size: str = "1.7B",
    device: str = "auto",
    speed: float = 1.0,
    ref_audio: str = None,
    ref_text: str = None,
    instruct: str = None,
) -> bytes:
    """
    Generate speech using Qwen3-TTS local model.

    Supports three modes based on arguments:
    - Custom voice (default): Use a preset speaker name
    - Voice clone: Provide ref_audio (path) to clone a voice
    - Voice design: Provide instruct (text description) to design a voice

    Args:
        text: Text to synthesize
        voice: Speaker name for custom voice mode
            (aiden, dylan, eric, ono_anna, ryan, serena, sohee, uncle_fu, vivian)
        language: Language (auto, chinese, english, japanese, korean, french, etc.)
        model_size: '0.6B' or '1.7B'
        device: 'auto', 'cuda', 'mps', 'cpu'
        speed: Speech speed (not directly supported, reserved)
        ref_audio: Path to reference audio for voice cloning
        ref_text: Transcript of reference audio (recommended for cloning)
        instruct: Natural language voice description for voice design mode

    Returns:
        WAV audio bytes
    """
    import numpy as np
    import soundfile as sf

    if ref_audio:
        model = _get_qwen3_model(model_size, "base", device)
        wavs, sr = model.generate_voice_clone(
            text=text,
            language=language,
            ref_audio=ref_audio,
            ref_text=ref_text,
        )
    elif instruct:
        model = _get_qwen3_model(model_size, "voice_design", device)
        wavs, sr = model.generate_voice_design(
            text=text,
            language=language,
            instruct=instruct,
        )
    else:
        model = _get_qwen3_model(model_size, "custom_voice", device)
        wavs, sr = model.generate_custom_voice(
            text=text,
            language=language,
            speaker=voice.lower().replace(" ", "_"),
        )

    if not wavs:
        raise ValueError("Qwen3-TTS generated no audio")

    wav_buffer = io.BytesIO()
    sf.write(wav_buffer, wavs[0], sr, format='WAV')
    wav_buffer.seek(0)
    return wav_buffer.read()

wav_to_pcm16(wav_data)

Extract PCM16 data from WAV. Returns (pcm_data, sample_rate).

Source code in npcpy/gen/audio_gen.py
def wav_to_pcm16(wav_data: bytes) -> tuple:
    """Extract PCM16 data from WAV. Returns (pcm_data, sample_rate)."""
    import struct

    if wav_data[:4] != b'RIFF' or wav_data[8:12] != b'WAVE':
        raise ValueError("Invalid WAV data")

    pos = 12
    sample_rate = 24000
    while pos < len(wav_data) - 8:
        chunk_id = wav_data[pos:pos+4]
        chunk_size = struct.unpack('<I', wav_data[pos+4:pos+8])[0]

        if chunk_id == b'fmt ':
            sample_rate = struct.unpack('<I', wav_data[pos+12:pos+16])[0]
        elif chunk_id == b'data':
            return wav_data[pos+8:pos+8+chunk_size], sample_rate

        pos += 8 + chunk_size

    raise ValueError("No data chunk found in WAV")

video_gen

generate_video_diffusers(prompt, model, npc=None, device='gpu', output_path='', num_inference_steps=5, num_frames=25, height=256, width=256)

Source code in npcpy/gen/video_gen.py
def generate_video_diffusers(
    prompt,
    model,
    npc=None,
    device="gpu",
    output_path="",
    num_inference_steps=5,
    num_frames=25,
    height=256,
    width=256,
):

    import torch
    from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
    import numpy as np
    import os 
    import cv2


    pipe = DiffusionPipeline.from_pretrained(
        "damo-vilab/text-to-video-ms-1.7b", torch_dtype=torch.float32
    ).to(device)
    pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)

    output = pipe(
        prompt,
        num_inference_steps=num_inference_steps,
        num_frames=num_frames,
        height=height,
        width=width,
    )

    def save_frames_to_video(frames, output_path, fps=8):
        """Handle the specific 5D array format (1, num_frames, H, W, 3) with proper type conversion"""

        if not (
            isinstance(frames, np.ndarray)
            and frames.ndim == 5
            and frames.shape[-1] == 3
        ):
            raise ValueError(
                f"Unexpected frame format. Expected 5D RGB array, got {frames.shape}"
            )


        frames = (frames[0] * 255).astype(np.uint8)  


        height, width = frames.shape[1:3]


        fourcc = cv2.VideoWriter_fourcc(*"mp4v")
        video_writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height))

        if not video_writer.isOpened():
            raise IOError(f"Could not open video writer for {output_path}")


        for frame in frames:
            video_writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))

        video_writer.release()
        print(f"Successfully saved {frames.shape[0]} frames to {output_path}")

    from npcpy.npc_sysenv import get_videos_dir
    _vid_dir = get_videos_dir()
    os.makedirs(_vid_dir, exist_ok=True)
    if output_path == "":
        output_path = os.path.join(_vid_dir, prompt[0:8] + ".mp4")
    save_frames_to_video(output.frames, output_path)
    return output_path

generate_video_veo3(prompt, model, negative_prompt='', output_path='')

Generate video using Google's Veo 3 API with synchronized audio.

Source code in npcpy/gen/video_gen.py
def generate_video_veo3(
    prompt: str,
    model: str, 
    negative_prompt: str = "",
    output_path: str = "",
):
    """
    Generate video using Google's Veo 3 API with synchronized audio.
    """
    import time
    import os
    from google import genai
    from google.genai import types
    api_key =os.environ.get('GEMINI_API_KEY')

    client = genai.Client(        api_key=api_key)

    config = types.GenerateVideosConfig()
    if negative_prompt:
        config.negative_prompt = negative_prompt

    operation = client.models.generate_videos(
        model=model,
        prompt=prompt,
        config=config,
    )

    while not operation.done:
        time.sleep(20)
        operation = client.operations.get(operation)

    generated_video = operation.result.generated_videos[0]

    from npcpy.npc_sysenv import get_videos_dir
    videos_dir = get_videos_dir()
    os.makedirs(videos_dir, exist_ok=True)
    if output_path == "":
        safe_prompt = "".join(c for c in prompt[:20] if c.isalnum() or c in (' ', '-', '_')).rstrip()
        output_path = os.path.join(videos_dir, safe_prompt.replace(" ", "_") + "_veo3.mp4")

    client.files.download(file=generated_video.video)
    generated_video.video.save(output_path)

    return output_path

response

HAS_OLLAMA = True module-attribute

TOKEN_COSTS = {'gpt-4o': (2.5, 10.0), 'gpt-4o-mini': (0.15, 0.6), 'gpt-4-turbo': (10.0, 30.0), 'gpt-3.5-turbo': (0.5, 1.5), 'gpt-5': (1.25, 10.0), 'gpt-5-mini': (0.25, 2.0), 'o1': (15.0, 60.0), 'o1-mini': (3.0, 12.0), 'o3': (10.0, 40.0), 'o3-mini': (1.1, 4.4), 'o4-mini': (1.1, 4.4), 'claude-3-5-sonnet': (3.0, 15.0), 'claude-3-opus': (15.0, 75.0), 'claude-3-haiku': (0.25, 1.25), 'claude-sonnet-4': (3.0, 15.0), 'claude-opus-4': (15.0, 75.0), 'claude-opus-4-5': (5.0, 25.0), 'claude-sonnet-4-5': (3.0, 15.0), 'claude-haiku-4': (0.8, 4.0), 'gemini-1.5-pro': (1.25, 5.0), 'gemini-1.5-flash': (0.075, 0.3), 'gemini-2.0-flash': (0.1, 0.4), 'gemini-2.5-pro': (1.25, 10.0), 'gemini-2.5-flash': (0.15, 0.6), 'gemini-3.1-pro': (2.0, 12.0), 'llama-3': (0.05, 0.08), 'llama-3.1': (0.05, 0.08), 'llama-3.2': (0.05, 0.08), 'llama-4': (0.05, 0.1), 'mixtral': (0.24, 0.24), 'deepseek-v3': (0.27, 1.1), 'deepseek-r1': (0.55, 2.19), 'mistral-large': (2.0, 6.0), 'mistral-small': (0.2, 0.6), 'grok-2': (2.0, 10.0), 'grok-3': (3.0, 15.0)} module-attribute

_AIRLLM_MLX_PATCHED = False module-attribute

_AIRLLM_MODEL_CACHE = {} module-attribute

logger = logging.getLogger(__name__) module-attribute

_patch_airllm_mlx_bias()

Monkey-patch airllm's MLX Attention/FeedForward to use bias=True. AirLLM hardcodes bias=False which fails for non-Llama architectures (e.g. Qwen2). Using bias=True is safe: MLX nn.Linear(bias=True) accepts weight-only updates, so Llama models (no bias in weights) still work correctly.

Source code in npcpy/gen/response.py
def _patch_airllm_mlx_bias():
    """
    Monkey-patch airllm's MLX Attention/FeedForward to use bias=True.
    AirLLM hardcodes bias=False which fails for non-Llama architectures (e.g. Qwen2).
    Using bias=True is safe: MLX nn.Linear(bias=True) accepts weight-only updates,
    so Llama models (no bias in weights) still work correctly.
    """
    global _AIRLLM_MLX_PATCHED
    if _AIRLLM_MLX_PATCHED:
        return
    try:
        import airllm.airllm_llama_mlx as mlx_mod
        import mlx.core as mx
        from mlx import nn

        class PatchedAttention(nn.Module):
            def __init__(self, args):
                super().__init__()
                self.args = args
                self.n_heads = args.n_heads
                self.n_kv_heads = args.n_kv_heads
                self.repeats = self.n_heads // self.n_kv_heads
                self.scale = args.head_dim ** -0.5
                self.wq = nn.Linear(args.dim, args.n_heads * args.head_dim, bias=True)
                self.wk = nn.Linear(args.dim, args.n_kv_heads * args.head_dim, bias=True)
                self.wv = nn.Linear(args.dim, args.n_kv_heads * args.head_dim, bias=True)
                self.wo = nn.Linear(args.n_heads * args.head_dim, args.dim, bias=True)
                self.rope = nn.RoPE(
                    args.head_dim, traditional=args.rope_traditional, base=args.rope_theta
                )

            def __call__(self, x, mask=None, cache=None):
                B, L, D = x.shape
                queries, keys, values = self.wq(x), self.wk(x), self.wv(x)
                queries = queries.reshape(B, L, self.n_heads, -1).transpose(0, 2, 1, 3)
                keys = keys.reshape(B, L, self.n_kv_heads, -1).transpose(0, 2, 1, 3)
                values = values.reshape(B, L, self.n_kv_heads, -1).transpose(0, 2, 1, 3)

                def repeat(a):
                    a = mx.concatenate([mx.expand_dims(a, 2)] * self.repeats, axis=2)
                    return a.reshape([B, self.n_heads, L, -1])
                keys, values = map(repeat, (keys, values))

                if cache is not None:
                    key_cache, value_cache = cache
                    queries = self.rope(queries, offset=key_cache.shape[2])
                    keys = self.rope(keys, offset=key_cache.shape[2])
                    keys = mx.concatenate([key_cache, keys], axis=2)
                    values = mx.concatenate([value_cache, values], axis=2)
                else:
                    queries = self.rope(queries)
                    keys = self.rope(keys)

                scores = (queries * self.scale) @ keys.transpose(0, 1, 3, 2)
                if mask is not None:
                    scores += mask
                weights = mx.softmax(scores.astype(mx.float32), axis=-1).astype(scores.dtype)
                output = (weights @ values).transpose(0, 2, 1, 3).reshape(B, L, -1)
                return self.wo(output), (keys, values)

        class PatchedFeedForward(nn.Module):
            def __init__(self, args):
                super().__init__()
                self.w1 = nn.Linear(args.dim, args.hidden_dim, bias=True)
                self.w2 = nn.Linear(args.hidden_dim, args.dim, bias=True)
                self.w3 = nn.Linear(args.dim, args.hidden_dim, bias=True)

            def __call__(self, x):
                return self.w2(nn.silu(self.w1(x)) * self.w3(x))

        mlx_mod.Attention = PatchedAttention
        mlx_mod.FeedForward = PatchedFeedForward
        _AIRLLM_MLX_PATCHED = True
        logger.debug("Patched airllm MLX classes for bias support")
    except Exception as e:
        logger.warning(f"Failed to patch airllm MLX bias support: {e}")

_require_ollama()

Raise a clear ImportError when the optional ollama package is absent.

Called at each entry point into the ollama provider so that users get a descriptive error instead of a bare NameError: name 'ollama' is not defined at an arbitrary call site.

Source code in npcpy/gen/response.py
def _require_ollama() -> None:
    """Raise a clear ImportError when the optional ``ollama`` package is absent.

    Called at each entry point into the ollama provider so that users get a
    descriptive error instead of a bare ``NameError: name 'ollama' is not
    defined`` at an arbitrary call site.
    """
    if not HAS_OLLAMA:
        raise ImportError(
            "The 'ollama' package is required for the ollama provider. "
            "Install it with: pip install ollama"
        )

calculate_cost(model, input_tokens, output_tokens)

Calculate cost in USD for a response.

Source code in npcpy/gen/response.py
def calculate_cost(model: str, input_tokens: int, output_tokens: int) -> float:
    """Calculate cost in USD for a response."""
    if not model:
        return 0.0

    model_key = model.split("/")[-1].lower()

    costs = None
    for key, cost in TOKEN_COSTS.items():
        if key in model_key or model_key in key:
            costs = cost
            break

    if not costs:
        return 0.0

    input_cost, output_cost = costs
    return (input_tokens * input_cost / 1_000_000) + (output_tokens * output_cost / 1_000_000)

compress_image(image_bytes, max_size=(800, 600))

Source code in npcpy/data/image.py
def compress_image(image_bytes, max_size=(800, 600)):

    buffer = io.BytesIO(image_bytes)
    img = Image.open(buffer)


    img.load()


    if img.mode == "RGBA":
        background = Image.new("RGB", img.size, (255, 255, 255))
        background.paste(img, mask=img.split()[3])
        img = background


    if img.size[0] > max_size[0] or img.size[1] > max_size[1]:
        img.thumbnail(max_size)


    out_buffer = io.BytesIO()
    img.save(out_buffer, format="JPEG", quality=95, optimize=False)
    return out_buffer.getvalue()

get_airllm_response(prompt=None, model=None, tools=None, tool_map=None, format=None, messages=None, auto_process_tool_calls=False, **kwargs)

Generate response using AirLLM for 70B+ model inference. Supports macOS (MLX backend) and Linux (CUDA backend with 4-bit compression).

Source code in npcpy/gen/response.py
def get_airllm_response(
    prompt: str = None,
    model: str = None,
    tools: list = None,
    tool_map: Dict = None,
    format: str = None,
    messages: List[Dict[str, str]] = None,
    auto_process_tool_calls: bool = False,
    **kwargs,
) -> Dict[str, Any]:
    """
    Generate response using AirLLM for 70B+ model inference.
    Supports macOS (MLX backend) and Linux (CUDA backend with 4-bit compression).
    """
    import platform
    is_macos = platform.system() == "Darwin"

    result = {
        "response": None,
        "messages": messages.copy() if messages else [],
        "raw_response": None,
        "tool_calls": [],
        "tool_results": []
    }

    try:
        from airllm import AutoModel
    except ImportError:
        result["response"] = ""
        result["error"] = "airllm not installed. Install with: pip install airllm"
        return result

    if is_macos:
        _patch_airllm_mlx_bias()

    if prompt:
        if result['messages'] and result['messages'][-1]["role"] == "user":
            result['messages'][-1]["content"] = prompt
        else:
            result['messages'].append({"role": "user", "content": prompt})

    if format == "json":
        json_instruction = """If you are returning a json object, begin directly with the opening {.
Do not include any additional markdown formatting or leading ```json tags in your response."""
        if result["messages"] and result["messages"][-1]["role"] == "user":
            result["messages"][-1]["content"] += "\n" + json_instruction

    model_name = model or "meta-llama/Meta-Llama-3.1-70B-Instruct"
    default_compression = None if is_macos else "4bit"
    compression = kwargs.get("compression", default_compression)
    max_tokens = kwargs.get("max_tokens", 256)
    temperature = kwargs.get("temperature", 0.7)

    hf_token = kwargs.get("hf_token")
    if not hf_token:
        hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
    if not hf_token:
        try:
            from huggingface_hub import HfFolder
            hf_token = HfFolder.get_token()
        except Exception:
            pass

    cache_key = f"{model_name}:{compression}"
    if cache_key not in _AIRLLM_MODEL_CACHE:
        load_kwargs = {"pretrained_model_name_or_path": model_name}
        if compression:
            load_kwargs["compression"] = compression
        if hf_token:
            load_kwargs["hf_token"] = hf_token
        for k in ["delete_original", "max_seq_len", "prefetching"]:
            if k in kwargs:
                load_kwargs[k] = kwargs[k]
        _AIRLLM_MODEL_CACHE[cache_key] = AutoModel.from_pretrained(**load_kwargs)

    air_model = _AIRLLM_MODEL_CACHE[cache_key]

    try:
        chat_text = air_model.tokenizer.apply_chat_template(
            result["messages"], tokenize=False, add_generation_prompt=True
        )
    except Exception:
        chat_text = "\n".join(
            f"{m['role']}: {m['content']}" for m in result["messages"]
        )
        chat_text += "\nassistant:"

    try:
        if is_macos:
            import mlx.core as mx
            tokens = air_model.tokenizer(
                chat_text, return_tensors="np", truncation=True, max_length=2048
            )
            output = air_model.generate(
                mx.array(tokens['input_ids']),
                max_new_tokens=max_tokens,
            )
            response_content = output if isinstance(output, str) else str(output)
        else:
            tokens = air_model.tokenizer(
                chat_text, return_tensors="pt", truncation=True, max_length=2048
            )
            gen_out = air_model.generate(
                tokens['input_ids'].cuda(),
                max_new_tokens=max_tokens,
            )
            output_ids = gen_out.sequences[0] if hasattr(gen_out, 'sequences') else gen_out[0]
            response_content = air_model.tokenizer.decode(output_ids, skip_special_tokens=True)
            input_text = air_model.tokenizer.decode(tokens['input_ids'][0], skip_special_tokens=True)
            if response_content.startswith(input_text):
                response_content = response_content[len(input_text):]

        response_content = response_content.strip()
        for stop_tok in ["<|im_end|>", "<|endoftext|>", "<|eot_id|>", "</s>"]:
            if stop_tok in response_content:
                response_content = response_content[:response_content.index(stop_tok)].strip()
    except Exception as e:
        logger.error(f"AirLLM inference error: {e}")
        result["error"] = f"AirLLM inference error: {str(e)}"
        result["response"] = ""
        return result

    result["response"] = response_content
    result["raw_response"] = response_content
    result["messages"].append({"role": "assistant", "content": response_content})

    if format == "json":
        try:
            if response_content.startswith("```json"):
                response_content = response_content.replace("```json", "").replace("```", "").strip()
            parsed_response = json.loads(response_content)
            result["response"] = parsed_response
        except json.JSONDecodeError:
            result["error"] = f"Invalid JSON response: {response_content}"

    return result

get_litellm_response(prompt=None, model=None, provider=None, images=None, tools=None, tool_choice=None, tool_map=None, think=None, format=None, messages=None, api_key=None, api_url=None, stream=False, attachments=None, auto_process_tool_calls=False, include_usage=False, **kwargs)

Source code in npcpy/gen/response.py
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
def get_litellm_response(
    prompt: str = None,
    model: str = None,
    provider: str = None,
    images: List[str] = None,
    tools: list = None,
    tool_choice: Dict = None,
    tool_map: Dict = None,
    think= None,
    format: Union[str, BaseModel] = None,
    messages: List[Dict[str, str]] = None,
    api_key: str = None,
    api_url: str = None,
    stream: bool = False,
    attachments: List[str] = None,
    auto_process_tool_calls: bool = False, 
    include_usage: bool = False,
    **kwargs,
) -> Dict[str, Any]:
    if not model:
        raise ValueError("No model specified. Please set a model in your NPC configuration or team settings.")
    result = {
        "response": None,
        "messages": messages.copy() if messages else [],
        "raw_response": None,
        "tool_calls": [],
        "tool_results":[],
    }
    if provider == "ollama":
        return get_ollama_response(
            prompt, 
            model, 
            images=images, 
            tools=tools, 
            tool_choice=tool_choice, 
            tool_map=tool_map,
            think=think,
            format=format, 
            messages=messages, 
            stream=stream, 
            attachments=attachments, 
            auto_process_tool_calls=auto_process_tool_calls, 
            **kwargs
        )
    elif provider == 'transformers':
        return get_transformers_response(
            prompt,
            model,
            images=images,
            tools=tools,
            tool_choice=tool_choice,
            tool_map=tool_map,
            think=think,
            format=format,
            messages=messages,
            stream=stream,
            attachments=attachments,
            auto_process_tool_calls=auto_process_tool_calls,
            **kwargs
        )
    elif provider == 'lora':
        print(f"🔧 LoRA provider detected, calling get_lora_response with model: {model}")
        result = get_lora_response(
            prompt=prompt,
            model=model,
            tools=tools,
            tool_map=tool_map,
            format=format,
            messages=messages,
            stream=stream,
            auto_process_tool_calls=auto_process_tool_calls,
            **kwargs
        )
        print(f"🔧 LoRA response: {result.get('response', 'NO RESPONSE')[:200] if result.get('response') else 'EMPTY'}")
        if result.get('error'):
            print(f"🔧 LoRA error: {result.get('error')}")
        return result
    elif provider == 'llamacpp':
        return get_llamacpp_response(
            prompt,
            model,
            images=images,
            tools=tools,
            tool_choice=tool_choice,
            tool_map=tool_map,
            think=think,
            format=format,
            messages=messages,
            stream=stream,
            attachments=attachments,
            auto_process_tool_calls=auto_process_tool_calls,
            **kwargs
        )
    elif provider == 'airllm':
        return get_airllm_response(
            prompt=prompt,
            model=model,
            tools=tools,
            tool_map=tool_map,
            format=format,
            messages=messages,
            auto_process_tool_calls=auto_process_tool_calls,
            **kwargs
        )
    elif provider == 'lmstudio' or (model and '.lmstudio' in str(model)):
        api_url = api_url or "http://127.0.0.1:1234/v1"
        provider = "openai"
        api_key = api_key or "lm-studio"
        if 'timeout' not in kwargs:
            kwargs['timeout'] = 300
    elif provider == 'llamacpp-server':
        api_url = api_url or "http://127.0.0.1:8080/v1"
        provider = "openai"
        api_key = api_key or "llamacpp"
        if 'timeout' not in kwargs:
            kwargs['timeout'] = 300
    elif provider == 'omlx':
        api_url = api_url or "http://127.0.0.1:8000/v1"
        provider = "openai"
        api_key = api_key or "omlx"
        if 'timeout' not in kwargs:
            kwargs['timeout'] = 300

    if attachments:
        for attachment in attachments:
            if os.path.exists(attachment):
                _, ext = os.path.splitext(attachment)
                ext = ext.lower()

                if ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp']:
                    if not images:
                        images = []
                    images.append(attachment)
                elif ext == '.pdf':
                    try:
                        from npcpy.data.load import load_pdf
                        pdf_data = load_pdf(attachment)
                        if pdf_data is not None:
                            if prompt:
                                prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data}..."
                            else:
                                prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data}..."

                    except Exception:
                        pass
                elif ext == '.csv':
                    try:
                        from npcpy.data.load import load_csv
                        csv_data = load_csv(attachment)
                        if csv_data is not None:
                            csv_sample = csv_data.head(10).to_string()
                            if prompt:
                                prompt += f"\n\nContent from CSV: {os.path.basename(attachment)} (first 10 rows):\n{csv_sample}"
                            else:
                                prompt = f"Content from CSV: {os.path.basename(attachment)} (first 10 rows):\n{csv_sample}"
                    except Exception:
                        pass
                else:
                    text_extensions = {'.txt', '.text', '.log', '.md', '.markdown', '.rst', '.json', '.yaml', '.yml', '.toml', '.ini', '.conf', '.cfg', '.xml', '.html', '.htm', '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.c', '.h', '.cpp', '.hpp', '.go', '.rs', '.rb', '.php', '.sh', '.bash', '.sql', '.css', '.scss'}
                    filename = os.path.basename(attachment)
                    if ext in text_extensions or ext == '':
                        try:
                            with open(attachment, 'r', encoding='utf-8', errors='replace') as f:
                                text_content = f.read()
                            max_chars = 50000
                            if len(text_content) > max_chars:
                                text_content = text_content[:max_chars] + f"\n\n... [truncated]"
                            if text_content.strip():
                                if prompt:
                                    prompt += f"\n\nContent from {filename}:\n```\n{text_content}\n```"
                                else:
                                    prompt = f"Content from {filename}:\n```\n{text_content}\n```"
                        except Exception:
                            pass

    if prompt:
        if result['messages'] and result['messages'][-1]["role"] == "user":
            if isinstance(messages[-1]["content"], str):
                result['messages'][-1]["content"] = prompt
            elif isinstance(result['messages'][-1]["content"], list):
                for i, item in enumerate(result['messages'][-1]["content"]):
                    if item.get("type") == "text":
                        result['messages'][-1]["content"][i]["text"] = prompt
                        break
                else:
                    result['messages'][-1]["content"].append({"type": "text", "text": prompt})
        else:
            result['messages'].append({"role": "user", "content": prompt})

    if format == "json" and not stream:
        json_instruction = """If you are a returning a json object, begin directly with the opening {.
            If you are returning a json array, begin directly with the opening [.
            Do not include any additional markdown formatting or leading
            ```json tags in your response. The item keys should be based on the ones provided
            by the user. Do not invent new ones."""

        if result["messages"] and result["messages"][-1]["role"] == "user":
            if isinstance(result["messages"][-1]["content"], list):
                result["messages"][-1]["content"].append({"type": "text", "text": json_instruction})
            elif isinstance(result["messages"][-1]["content"], str):
                result["messages"][-1]["content"] += "\n" + json_instruction

    if format == "yaml" and not stream:
        yaml_instruction = """Return your response as valid YAML. Do not include ```yaml markdown tags.
            For multi-line strings like code, use the literal block scalar (|) syntax:
            code: |
              your code here
              more lines here
            The keys should be based on the ones requested by the user. Do not invent new ones."""

        if result["messages"] and result["messages"][-1]["role"] == "user":
            if isinstance(result["messages"][-1]["content"], list):
                result["messages"][-1]["content"].append({"type": "text", "text": yaml_instruction})
            elif isinstance(result["messages"][-1]["content"], str):
                result["messages"][-1]["content"] += "\n" + yaml_instruction

    if images:
        last_user_idx = -1
        for i, msg in enumerate(result["messages"]):
            if msg["role"] == "user":
                last_user_idx = i
        if last_user_idx == -1:
            result["messages"].append({"role": "user", "content": []})
            last_user_idx = len(result["messages"]) - 1
        if isinstance(result["messages"][last_user_idx]["content"], str):

            result["messages"][last_user_idx]["content"] = [{"type": "text", 
                                                             "text": result["messages"][last_user_idx]["content"]
                                                             }]

        elif not isinstance(result["messages"][last_user_idx]["content"], list):
            result["messages"][last_user_idx]["content"] = []
        for image_path in images:
            with open(image_path, "rb") as image_file:
                image_data = base64.b64encode(compress_image(image_file.read())).decode("utf-8")
                result["messages"][last_user_idx]["content"].append(
                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
                )



    result["messages"] = sanitize_messages(result["messages"])
    api_params = {"messages": result["messages"]}

    if include_usage:
      litellm.include_cost_in_streaming_usage = True
      api_params['stream_options'] = {"include_usage": True}

    if api_url is not None and ('openai-like' in provider or provider == "openai-like" or provider == "openai"):
        api_params["api_base"] = api_url
        provider = "openai"


    if provider =='enpisi' and api_url is None:
        api_params['api_base'] = 'https://api.enpisi.com'
        if api_key is None:
            api_key = os.environ.get('NPC_STUDIO_LICENSE_KEY')
            api_params['api_key'] = api_key
        if '-npc' in model: 
            model = model.split('-npc')[0]
        provider = "openai"

    if isinstance(format, type) and issubclass(format, BaseModel):
        api_params["response_format"] = format
    if model is None:
        raise ValueError("No model specified. Please set a model in your NPC configuration or team settings.")
    if provider is None:
        raise ValueError("No provider specified. Please set a provider in your NPC configuration or team settings.")

    if "api_base" in api_params and provider == "openai":
        api_params["model"] = f"openai/{model}"
    elif "/" not in model or model.startswith("/"):
        api_params["model"] = f"{provider}/{model}"
    else:
        api_params["model"] = model
    if api_key is not None: 
        api_params["api_key"] = api_key
    if tools: 
        api_params["tools"] = tools
    if tool_choice: 
        api_params["tool_choice"] = tool_choice

    if kwargs:
        for key, value in kwargs.items():
            if key in [
                "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens",
                 "extra_headers", "parallel_tool_calls",
                "response_format", "user", "timeout", "think", "thinking", "reasoning_effort",
            ]:
                # Handle temperature/top_p conflict for Claude models
                if key == "temperature" and "claude" in str(api_params.get("model", "")).lower():
                    api_params[key] = value
                elif key == "top_p" and "claude" in str(api_params.get("model", "")).lower():
                    # Only add top_p for Claude if temperature is not provided
                    if "temperature" not in kwargs:
                        api_params[key] = value
                else:
                    api_params[key] = value

    if not auto_process_tool_calls or not (tools and tool_map):
        api_params["stream"] = stream
        resp = completion(**api_params)
        result["raw_response"] = resp

        if hasattr(resp, 'usage') and resp.usage:
            result["usage"] = {
                "input_tokens": getattr(resp.usage, 'prompt_tokens', 0) or 0,
                "output_tokens": getattr(resp.usage, 'completion_tokens', 0) or 0,
            }
        elif hasattr(resp, 'prompt_eval_count'):
            result["usage"] = {
                "input_tokens": getattr(resp, 'prompt_eval_count', 0) or 0,
                "output_tokens": getattr(resp, 'eval_count', 0) or 0,
            }

        if stream:
            result["response"] = resp
            return result
        else:

            llm_response = resp.choices[0].message.content
            result["response"] = llm_response
            assistant_msg = {"role": "assistant", "content": llm_response}
            if hasattr(resp.choices[0].message, 'tool_calls') and resp.choices[0].message.tool_calls:
                raw_tcs = resp.choices[0].message.tool_calls
                result["tool_calls"] = raw_tcs
                tc_dicts = []
                for tc in raw_tcs:
                    if isinstance(tc, dict):
                        tc_dicts.append(tc)
                    else:
                        tc_dicts.append({
                            "id": getattr(tc, "id", str(uuid.uuid4())),
                            "type": "function",
                            "function": {
                                "name": getattr(tc.function, "name", "") if hasattr(tc, "function") else "",
                                "arguments": getattr(tc.function, "arguments", "{}") if hasattr(tc, "function") else "{}"
                            }
                        })
                assistant_msg["tool_calls"] = tc_dicts
            result["messages"].append(assistant_msg)
            if format == "json":
                try:
                    if isinstance(llm_response, str):
                        llm_response = llm_response.strip()

                        if '```json' in llm_response:
                            start = llm_response.find('```json') + 7
                            end = llm_response.rfind('```')
                            if end > start:
                                llm_response = llm_response[start:end].strip()

                        first_brace = llm_response.find('{')
                        first_bracket = llm_response.find('[')

                        if first_brace == -1 and first_bracket == -1:
                            result["response"] = {}
                            result["error"] = "No JSON found in response"
                            return result

                        if first_brace != -1 and (first_bracket == -1 or first_brace < first_bracket):
                            llm_response = llm_response[first_brace:]
                            last_brace = llm_response.rfind('}')
                            if last_brace != -1:
                                llm_response = llm_response[:last_brace+1]
                        else:
                            llm_response = llm_response[first_bracket:]
                            last_bracket = llm_response.rfind(']')
                            if last_bracket != -1:
                                llm_response = llm_response[:last_bracket+1]

                        parsed_json = json.loads(llm_response, strict=False)

                        if "json" in parsed_json:
                            result["response"] = parsed_json["json"]
                        else:
                            result["response"] = parsed_json

                except (json.JSONDecodeError, TypeError) as e:
                    logger.debug(f"JSON parsing error: {str(e)}, raw response: {llm_response[:500]}")
                    result["response"] = {}
                    result["error"] = "Invalid JSON response"

            if format == "yaml":
                try:
                    if isinstance(llm_response, str):
                        llm_response = llm_response.strip()

                        if '```yaml' in llm_response:
                            start = llm_response.find('```yaml') + 7
                            end = llm_response.rfind('```')
                            if end > start:
                                llm_response = llm_response[start:end].strip()
                        elif '```' in llm_response:
                            start = llm_response.find('```') + 3
                            newline = llm_response.find('\n', start)
                            if newline != -1:
                                start = newline + 1
                            end = llm_response.rfind('```')
                            if end > start:
                                llm_response = llm_response[start:end].strip()

                        parsed_yaml = yaml.safe_load(llm_response)
                        result["response"] = parsed_yaml

                except (yaml.YAMLError, TypeError) as e:
                    logger.debug(f"YAML parsing error: {str(e)}, raw response: {llm_response[:500]}")
                    result["response"] = {}
                    result["error"] = "Invalid YAML response"

            return result



    initial_api_params = api_params.copy()
    initial_api_params["stream"] = False

    try:
        resp = completion(**initial_api_params)
    except Exception as e:
        logger.error(f"litellm completion() failed: {type(e).__name__}: {e}")
        result["error"] = str(e)
        result["response"] = f"LLM call failed: {e}"
        return result

    result["raw_response"] = resp

    if hasattr(resp, 'usage') and resp.usage:
        result["usage"] = {
            "input_tokens": getattr(resp.usage, 'prompt_tokens', 0) or 0,
            "output_tokens": getattr(resp.usage, 'completion_tokens', 0) or 0,
        }

    if not resp.choices:
        result["response"] = "No response from model"
        return result

    has_tool_calls = hasattr(resp.choices[0].message, 'tool_calls') and resp.choices[0].message.tool_calls

    if has_tool_calls:
        result["tool_calls"] = resp.choices[0].message.tool_calls

        processed_result = process_tool_calls(result,
                                              tool_map,
                                              model,
                                              provider,
                                              result["messages"],
                                              stream=False,
                                              tools=tools)

        clean_messages = []
        tool_results_summary = []

        for msg in processed_result["messages"]:
            role = msg.get('role', '')
            if role == 'assistant' and 'tool_calls' in msg:
                continue
            elif role == 'tool':
                content = msg.get('content', '')
                if len(content) > 2000:
                    content = content[:2000] + "... (truncated)"
                tool_results_summary.append(content)
            else:
                clean_messages.append(msg)

        if tool_results_summary:
            clean_messages.append({
                "role": "assistant",
                "content": "I executed the requested tools. Here are the results:\n\n" + "\n\n".join(tool_results_summary)
            })

        clean_messages.append({
            "role": "user",
            "content": "Based on the tool results above, provide a brief summary of what happened. Do NOT output any code - the tool has already executed. Just describe the results concisely."
        })

        final_api_params = api_params.copy()
        final_api_params["messages"] = clean_messages
        final_api_params["stream"] = stream
        if "tools" in final_api_params:
            del final_api_params["tools"]
        if "tool_choice" in final_api_params:
            del final_api_params["tool_choice"]

        final_resp = completion(**final_api_params)

        if stream:
            processed_result["response"] = final_resp
        else:
            if final_resp.choices:
                final_content = final_resp.choices[0].message.content
                processed_result["response"] = final_content
                processed_result["messages"].append({"role": "assistant", "content": final_content})
            else:
                if tool_results_summary:
                    fallback_content = "\n\n".join(tool_results_summary)
                else:
                    fallback_content = "Tool executed successfully."
                processed_result["response"] = fallback_content
                processed_result["messages"].append({"role": "assistant", "content": fallback_content})

        return processed_result


    else:
        llm_response = resp.choices[0].message.content
        result["messages"].append({"role": "assistant", "content": llm_response})

        if stream:
            def string_chunk_generator():
                chunk_size = 1
                for i, char in enumerate(llm_response):
                    yield type('MockChunk', (), {
                        'id': f'mock-chunk-{i}',
                        'object': 'chat.completion.chunk',
                        'created': int(time.time()),
                        'model': model or 'unknown',
                        'choices': [type('Choice', (), {
                            'index': 0,
                            'delta': type('Delta', (), {
                                'content': char,
                                'role': 'assistant' if i == 0 else None
                            })(),
                            'finish_reason': 'stop' if i == len(llm_response) - 1 else None
                        })()]
                    })()

            result["response"] = string_chunk_generator()
        else:
            result["response"] = llm_response
    return result            

get_llamacpp_response(prompt=None, model=None, images=None, tools=None, tool_choice=None, tool_map=None, think=None, format=None, messages=None, stream=False, attachments=None, auto_process_tool_calls=False, **kwargs)

Generate response using llama-cpp-python for local GGUF/GGML files.

Source code in npcpy/gen/response.py
def get_llamacpp_response(
    prompt: str = None,
    model: str = None,
    images: List[str] = None,
    tools: list = None,
    tool_choice: Dict = None,
    tool_map: Dict = None,
    think=None,
    format: Union[str, BaseModel] = None,
    messages: List[Dict[str, str]] = None,
    stream: bool = False,
    attachments: List[str] = None,
    auto_process_tool_calls: bool = False,
    **kwargs,
) -> Dict[str, Any]:
    """
    Generate response using llama-cpp-python for local GGUF/GGML files.
    """
    try:
        from llama_cpp import Llama
    except ImportError:
        return {
            "response": "",
            "messages": messages or [],
            "error": "llama-cpp-python not installed. Install with: pip install llama-cpp-python"
        }

    result = {
        "response": None,
        "messages": messages.copy() if messages else [],
        "raw_response": None,
        "tool_calls": [],
        "tool_results": []
    }

    if prompt:
        if messages and messages[-1]["role"] == "user":
            messages[-1]["content"] = prompt
        else:
            if not messages:
                messages = []
            messages.append({"role": "user", "content": prompt})

    try:
        n_ctx = kwargs.get("n_ctx", 32768)
        n_gpu_layers = kwargs.get("n_gpu_layers", -1)

        llm = Llama(
            model_path=model,
            n_ctx=n_ctx,
            n_gpu_layers=n_gpu_layers,
            verbose=False
        )

        params = {
            "messages": messages,
            "stream": stream,
        }
        if kwargs.get("temperature") is not None:
            params["temperature"] = kwargs["temperature"]
        if kwargs.get("max_tokens"):
            params["max_tokens"] = kwargs["max_tokens"]
        if kwargs.get("top_p") is not None:
            params["top_p"] = kwargs["top_p"]
        if kwargs.get("top_k") is not None:
            params["top_k"] = kwargs["top_k"]
        if kwargs.get("stop"):
            params["stop"] = kwargs["stop"]

        if stream:
            response = llm.create_chat_completion(**params)

            def generate():
                for chunk in response:
                    yield chunk

            result["response"] = generate()
        else:
            response = llm.create_chat_completion(**params)
            result["raw_response"] = response

            if response.get("choices"):
                content = response["choices"][0].get("message", {}).get("content", "")
                result["response"] = content
                result["messages"].append({"role": "assistant", "content": content})

            if response.get("usage"):
                result["usage"] = {
                    "input_tokens": response["usage"].get("prompt_tokens", 0),
                    "output_tokens": response["usage"].get("completion_tokens", 0),
                }

    except Exception as e:
        result["error"] = f"llama.cpp error: {str(e)}"
        result["response"] = ""

    return result

get_lora_response(prompt=None, model=None, tools=None, tool_map=None, format=None, messages=None, stream=False, auto_process_tool_calls=False, **kwargs)

Generate response using a LoRA adapter on top of a base model. The adapter path should contain adapter_config.json with base_model_name_or_path.

Source code in npcpy/gen/response.py
def get_lora_response(
    prompt: str = None,
    model: str = None,
    tools: list = None,
    tool_map: Dict = None,
    format: str = None,
    messages: List[Dict[str, str]] = None,
    stream: bool = False,
    auto_process_tool_calls: bool = False,
    **kwargs,
) -> Dict[str, Any]:
    """
    Generate response using a LoRA adapter on top of a base model.
    The adapter path should contain adapter_config.json with base_model_name_or_path.
    """
    print(f"🎯 get_lora_response called with model={model}, prompt={prompt[:50] if prompt else 'None'}...")

    result = {
        "response": None,
        "messages": messages.copy() if messages else [],
        "raw_response": None,
        "tool_calls": [],
        "tool_results": []
    }

    try:
        import torch
        from transformers import AutoTokenizer, AutoModelForCausalLM
        from peft import PeftModel
        print("🎯 Successfully imported torch, transformers, peft")
    except ImportError as e:
        print(f"🎯 Import error: {e}")
        return {
            "response": "",
            "messages": messages or [],
            "error": f"Missing dependencies for LoRA. Install with: pip install transformers peft torch. Error: {e}"
        }

    adapter_path = os.path.expanduser(model)
    adapter_config_path = os.path.join(adapter_path, 'adapter_config.json')

    if not os.path.exists(adapter_config_path):
        return {
            "response": "",
            "messages": messages or [],
            "error": f"No adapter_config.json found at {adapter_path}"
        }

    try:
        with open(adapter_config_path, 'r') as f:
            adapter_config = json.load(f)
        base_model_id = adapter_config.get('base_model_name_or_path')
        if not base_model_id:
            return {
                "response": "",
                "messages": messages or [],
                "error": "adapter_config.json missing base_model_name_or_path"
            }
    except Exception as e:
        return {
            "response": "",
            "messages": messages or [],
            "error": f"Failed to read adapter config: {e}"
        }

    if prompt:
        if result['messages'] and result['messages'][-1]["role"] == "user":
            result['messages'][-1]["content"] = prompt
        else:
            result['messages'].append({"role": "user", "content": prompt})

    if format == "json":
        json_instruction = """If you are returning a json object, begin directly with the opening {.
Do not include any additional markdown formatting or leading ```json tags in your response."""
        if result["messages"] and result["messages"][-1]["role"] == "user":
            result["messages"][-1]["content"] += "\n" + json_instruction

    try:
        logger.info(f"Loading base model: {base_model_id}")
        tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=True)
        base_model = AutoModelForCausalLM.from_pretrained(
            base_model_id,
            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
            device_map="auto" if torch.cuda.is_available() else None,
            trust_remote_code=True
        )

        if tokenizer.pad_token is None:
            tokenizer.pad_token = tokenizer.eos_token

        logger.info(f"Loading LoRA adapter: {adapter_path}")
        model_with_adapter = PeftModel.from_pretrained(base_model, adapter_path)

        chat_text = tokenizer.apply_chat_template(
            result["messages"],
            tokenize=False,
            add_generation_prompt=True
        )
        device = next(model_with_adapter.parameters()).device
        inputs = tokenizer(chat_text, return_tensors="pt", padding=True, truncation=True)
        inputs = {k: v.to(device) for k, v in inputs.items()}

        max_new_tokens = kwargs.get("max_tokens", 512)
        temperature = kwargs.get("temperature", 0.7)

        with torch.no_grad():
            outputs = model_with_adapter.generate(
                **inputs,
                max_new_tokens=max_new_tokens,
                temperature=temperature,
                do_sample=True,
                pad_token_id=tokenizer.eos_token_id,
            )

        response_content = tokenizer.decode(
            outputs[0][inputs['input_ids'].shape[1]:],
            skip_special_tokens=True
        ).strip()

        result["response"] = response_content
        result["raw_response"] = response_content
        result["messages"].append({"role": "assistant", "content": response_content})

        if format == "json":
            try:
                if response_content.startswith("```json"):
                    response_content = response_content.replace("```json", "").replace("```", "").strip()
                parsed_response = json.loads(response_content)
                result["response"] = parsed_response
            except json.JSONDecodeError:
                result["error"] = f"Invalid JSON response: {response_content}"

    except Exception as e:
        logger.error(f"LoRA inference error: {e}")
        result["error"] = f"LoRA inference error: {str(e)}"
        result["response"] = ""

    return result

get_model_context_window(model, provider=None)

Get the context window size (max input tokens) for a model.

Uses litellm's model info database. Falls back to provider-specific queries (e.g. ollama show) when litellm doesn't have the model.

Returns 0 if the context window cannot be determined.

Source code in npcpy/gen/response.py
def get_model_context_window(model: str, provider: str = None) -> int:
    """Get the context window size (max input tokens) for a model.

    Uses litellm's model info database. Falls back to provider-specific
    queries (e.g. ollama show) when litellm doesn't have the model.

    Returns 0 if the context window cannot be determined.
    """
    if not model:
        return 0

    # Try litellm first - it has a comprehensive model database
    try:
        info = litellm.get_model_info(model)
        ctx = info.get("max_input_tokens") or info.get("max_tokens") or 0
        if ctx > 0:
            return ctx
    except Exception:
        pass

    # Try with provider prefix if given
    if provider:
        try:
            prefixed = f"{provider}/{model}"
            info = litellm.get_model_info(prefixed)
            ctx = info.get("max_input_tokens") or info.get("max_tokens") or 0
            if ctx > 0:
                return ctx
        except Exception:
            pass

    # Fallback: query ollama directly for local models
    resolved_provider = provider
    if not resolved_provider:
        try:
            resolved_provider = lookup_provider(model)
        except Exception:
            pass

    if resolved_provider == "ollama":
        try:
            info = ollama.show(model)
            params = info.get("model_info", {})
            for key, val in params.items():
                if "context_length" in key:
                    return int(val)
        except Exception:
            pass
        # ollama default
        return int(os.environ.get("NPCSH_OLLAMA_NUM_CTX", 32768))

    return 0

get_ollama_response(prompt, model, images=None, tools=None, tool_choice=None, tool_map=None, think=None, format=None, messages=None, stream=False, attachments=None, auto_process_tool_calls=False, **kwargs)

Generates a response using the Ollama API, supporting both streaming and non-streaming.

Source code in npcpy/gen/response.py
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
def get_ollama_response(
    prompt: str,
    model: str,
    images: List[str] = None,
    tools: list = None,
    tool_choice: Dict = None,
    tool_map: Dict = None,
    think= None ,
    format: Union[str, BaseModel] = None,
    messages: List[Dict[str, str]] = None,
    stream: bool = False,
    attachments: List[str] = None,
    auto_process_tool_calls: bool = False,
    **kwargs,
) -> Dict[str, Any]:
    """
    Generates a response using the Ollama API, supporting both streaming and non-streaming.
    """
    _require_ollama()

    options = {}

    # Set num_ctx from environment or kwargs (default 32768).
    # Ollama defaults to 4096 which is too small for multi-turn tool use.
    num_ctx = int(os.environ.get("NPCSH_OLLAMA_NUM_CTX", 0)) or kwargs.pop("num_ctx", 32768)
    options["num_ctx"] = num_ctx

    image_paths = []
    if images:
        image_paths.extend(images)

    if attachments:
        for attachment in attachments:
            if os.path.exists(attachment):
                _, ext = os.path.splitext(attachment)
                ext = ext.lower()

                if ext in ['.jpg', '.jpeg', '.png', '.gif', '.bmp']:
                    image_paths.append(attachment)
                elif ext == '.pdf':
                    try:
                        from npcpy.data.load import load_pdf
                        pdf_data = load_pdf(attachment)
                        if pdf_data is not None:
                            if prompt:
                                prompt += f"\n\nContent from PDF: {os.path.basename(attachment)}\n{pdf_data[:5000]}..."
                            else:
                                prompt = f"Content from PDF: {os.path.basename(attachment)}\n{pdf_data[:5000]}..."
                    except Exception:
                        pass
                elif ext == '.csv':
                    try:
                        from npcpy.data.load import load_csv
                        csv_data = load_csv(attachment)
                        if csv_data is not None:
                            csv_sample = csv_data.head(100).to_string()
                            if prompt:
                                prompt += f"\n\nContent from CSV: {os.path.basename(attachment)} (first 100 rows):\n{csv_sample} \n csv description: {csv_data.describe()}"
                            else:
                                prompt = f"Content from CSV: {os.path.basename(attachment)} (first 100 rows):\n{csv_sample} \n csv description: {csv_data.describe()}"
                    except Exception:
                        pass
                else:
                    text_extensions = {'.txt', '.text', '.log', '.md', '.markdown', '.rst', '.json', '.yaml', '.yml', '.toml', '.ini', '.conf', '.cfg', '.xml', '.html', '.htm', '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.c', '.h', '.cpp', '.hpp', '.go', '.rs', '.rb', '.php', '.sh', '.bash', '.sql', '.css', '.scss'}
                    filename = os.path.basename(attachment)
                    if ext in text_extensions or ext == '':
                        try:
                            with open(attachment, 'r', encoding='utf-8', errors='replace') as f:
                                text_content = f.read()
                            max_chars = 50000
                            if len(text_content) > max_chars:
                                text_content = text_content[:max_chars] + f"\n\n... [truncated]"
                            if text_content.strip():
                                if prompt:
                                    prompt += f"\n\nContent from {filename}:\n```\n{text_content}\n```"
                                else:
                                    prompt = f"Content from {filename}:\n```\n{text_content}\n```"
                        except Exception:
                            pass

    if prompt:
        if messages and messages[-1]["role"] == "user":
            if isinstance(messages[-1]["content"], str):
                messages[-1]["content"] = prompt
            elif isinstance(messages[-1]["content"], list):
                for i, item in enumerate(messages[-1]["content"]):
                    if item.get("type") == "text":
                        messages[-1]["content"][i]["text"] = prompt
                        break
                else:
                    messages[-1]["content"].append({"type": "text", "text": prompt})
        else:
            if not messages:
                messages = []
            messages.append({"role": "user", "content": prompt})
    if format == "json" and not stream:
        json_instruction = """If you are a returning a json object, begin directly with the opening {.
            If you are returning a json array, begin directly with the opening [.
            Do not include any additional markdown formatting or leading
            ```json tags in your response. The item keys should be based on the ones provided
            by the user. Do not invent new ones."""

        if messages and messages[-1]["role"] == "user":
            if isinstance(messages[-1]["content"], list):
                messages[-1]["content"].append({
                    "type": "text",
                    "text": json_instruction
                })
            elif isinstance(messages[-1]["content"], str):
                messages[-1]["content"] += "\n" + json_instruction

    if format == "yaml" and not stream:
        yaml_instruction = """Return your response as valid YAML. Do not include ```yaml markdown tags.
            For multi-line strings like code, use the literal block scalar (|) syntax:
            code: |
              your code here
              more lines here
            The keys should be based on the ones requested by the user. Do not invent new ones."""

        if messages and messages[-1]["role"] == "user":
            if isinstance(messages[-1]["content"], list):
                messages[-1]["content"].append({
                    "type": "text",
                    "text": yaml_instruction
                })
            elif isinstance(messages[-1]["content"], str):
                messages[-1]["content"] += "\n" + yaml_instruction

    if image_paths:
        last_user_idx = -1
        for i, msg in enumerate(messages):
            if msg["role"] == "user":
                last_user_idx = i
        if last_user_idx == -1:
            messages.append({"role": "user", "content": ""})
            last_user_idx = len(messages) - 1
        messages[last_user_idx]["images"] = image_paths

    for msg in messages:
        if msg.get("tool_calls"):
            for tc in msg["tool_calls"]:
                if tc.get("function") and isinstance(tc["function"].get("arguments"), str):
                    try:
                        tc["function"]["arguments"] = json.loads(tc["function"]["arguments"])
                    except (json.JSONDecodeError, TypeError):
                        tc["function"]["arguments"] = {}

    api_params = {
        "model": model,
        "messages": messages,
        "stream": stream if not (tools and tool_map and auto_process_tool_calls) else False,
    }

    if tools:
        api_params["tools"] = tools
        if tool_choice:
            options["tool_choice"] = tool_choice

    if think is not None:
        api_params['think'] = think

    if isinstance(format, type) and not stream:
        api_params["format"] = format.model_json_schema()
    elif isinstance(format, str) and format == "json" and not stream:
        api_params["format"] = "json"

    for key, value in kwargs.items():
        if key in [
            "stop",
            "temperature",
            "top_p",
            "top_k",
            "max_tokens",
            "max_completion_tokens",
            "extra_headers",
            "parallel_tool_calls",
            "response_format",
            "user",
        ]:
            options[key] = value

    result = {
        "response": None,
        "messages": messages.copy(),
        "raw_response": None,
        "tool_calls": [], 
        "tool_results": []
    }




    api_params["messages"] = sanitize_messages(api_params["messages"])
    result["messages"] = api_params["messages"]

    if not auto_process_tool_calls or not (tools and tool_map):
        res = ollama.chat(**api_params, options=options)
        result["raw_response"] = res

        if stream:
            result["response"] = res
            return result

        # Extract usage if available
        if hasattr(res, 'prompt_eval_count') or 'prompt_eval_count' in res:
            input_tokens = getattr(res, 'prompt_eval_count', None) or res.get('prompt_eval_count', 0) or 0
            output_tokens = getattr(res, 'eval_count', None) or res.get('eval_count', 0) or 0
            result["usage"] = {
                "input_tokens": input_tokens,
                "output_tokens": output_tokens,
            }

        # Always extract response content
        message = res.get("message", {})
        response_content = message.get("content", "")
        result["response"] = response_content

        assistant_msg = {"role": "assistant", "content": response_content}
        if message.get('tool_calls'):
            result["tool_calls"] = message['tool_calls']
            assistant_msg["tool_calls"] = message['tool_calls']
        result["messages"].append(assistant_msg)

        if format == "json":
            try:
                if isinstance(response_content, str):
                    if response_content.startswith("```json"):
                        response_content = (
                            response_content.replace("```json", "")
                            .replace("```", "")
                            .strip()
                        )
                    parsed_response = json.loads(response_content)
                    result["response"] = parsed_response
            except json.JSONDecodeError:
                result["error"] = f"Invalid JSON response: {response_content}"

        if format == "yaml":
            try:
                if isinstance(response_content, str):
                    if response_content.startswith("```yaml"):
                        response_content = (
                            response_content.replace("```yaml", "")
                            .replace("```", "")
                            .strip()
                        )
                    parsed_response = yaml.safe_load(response_content)
                    result["response"] = parsed_response
            except yaml.YAMLError:
                result["error"] = f"Invalid YAML response: {response_content}"

        return result

    logger.debug(f"ollama api_params: {api_params}")
    res = ollama.chat(**api_params, options=options)
    result["raw_response"] = res



    message = res.get("message", {})
    response_content = message.get("content", "")


    if message.get('tool_calls'):


        result["tool_calls"] = message['tool_calls']

        response_for_processing = {
            "response": response_content,
            "raw_response": res,
            "messages": messages,
            "tool_calls": message['tool_calls']
        }


        processed_result = process_tool_calls(response_for_processing,
                                              tool_map, model,
                                              'ollama',
                                              messages,
                                              stream=False,
                                              tools=tools)


        clean_messages = []
        tool_results_summary = []

        for msg in processed_result["messages"]:
            role = msg.get('role', '')
            if role == 'assistant' and 'tool_calls' in msg:
                continue
            elif role == 'tool':
                content = msg.get('content', '')
                if len(content) > 2000:
                    content = content[:2000] + "... (truncated)"
                tool_results_summary.append(content)
            else:
                clean_messages.append(msg)

        if tool_results_summary:
            clean_messages.append({
                "role": "assistant",
                "content": "I executed the requested tools. Here are the results:\n\n" + "\n\n".join(tool_results_summary)
            })

        clean_messages.append({
            "role": "user",
            "content": "Based on the tool results above, provide a brief summary of what happened. Do NOT output any code - the tool has already executed. Just describe the results concisely."
        })

        final_api_params = {
            "model": model,
            "messages": clean_messages,
            "stream": stream,
        }

        if stream:
            final_stream = ollama.chat(**final_api_params, options=options)
            processed_result["response"] = final_stream
        else:
            final_resp = ollama.chat(**final_api_params, options=options)
            final_message = final_resp.get("message", {})
            final_content = final_message.get("content", "")
            if final_content:
                processed_result["response"] = final_content
                processed_result["messages"].append({"role": "assistant", "content": final_content})
            elif tool_results_summary:
                processed_result["response"] = "\n\n".join(tool_results_summary)
            else:
                processed_result["response"] = "Tool executed successfully."

        return processed_result


    else:
        result["response"] = response_content
        result["messages"].append({"role": "assistant", "content": response_content})

        if stream:

            stream_api_params = {
                "model": model,
                "messages": messages,
                "stream": True,
            }
            if tools:
                stream_api_params["tools"] = tools

            result["response"] = ollama.chat(**stream_api_params, options=options)
        else:

            if format == "json":
                try:
                    llm_response = response_content
                    if isinstance(llm_response, str):
                        llm_response = llm_response.strip()

                        if '```json' in llm_response:
                            start = llm_response.find('```json') + 7
                            end = llm_response.rfind('```')
                            if end > start:
                                llm_response = llm_response[start:end].strip()

                        first_brace = llm_response.find('{')
                        first_bracket = llm_response.find('[')

                        if first_brace == -1 and first_bracket == -1:
                            result["response"] = {}
                            result["error"] = "No JSON found in response"
                            return result

                        if first_brace != -1 and (first_bracket == -1 or first_brace < first_bracket):
                            llm_response = llm_response[first_brace:]
                            last_brace = llm_response.rfind('}')
                            if last_brace != -1:
                                llm_response = llm_response[:last_brace+1]
                        else:
                            llm_response = llm_response[first_bracket:]
                            last_bracket = llm_response.rfind(']')
                            if last_bracket != -1:
                                llm_response = llm_response[:last_bracket+1]

                        parsed_json = json.loads(llm_response, strict=False)

                        if "json" in parsed_json:
                            result["response"] = parsed_json["json"]
                        else:
                            result["response"] = parsed_json

                except (json.JSONDecodeError, TypeError) as e:
                    logger.debug(f"JSON parsing error: {str(e)}, raw response: {llm_response[:500]}")
                    result["response"] = {}
                    result["error"] = "Invalid JSON response"

            if format == "yaml":
                try:
                    if isinstance(llm_response, str):
                        llm_response = llm_response.strip()

                        if '```yaml' in llm_response:
                            start = llm_response.find('```yaml') + 7
                            end = llm_response.rfind('```')
                            if end > start:
                                llm_response = llm_response[start:end].strip()

                        parsed_yaml = yaml.safe_load(llm_response)
                        result["response"] = parsed_yaml

                except (yaml.YAMLError, TypeError) as e:
                    logger.debug(f"YAML parsing error: {str(e)}, raw response: {llm_response[:500]}")
                    result["response"] = {}
                    result["error"] = "Invalid YAML response"

        return result

get_system_message(npc, team=None, tool_capable=False)

Source code in npcpy/npc_sysenv.py
def get_system_message(npc, team=None, tool_capable=False) -> str:

    if npc is None:
        return "You are a helpful assistant"
    if npc.plain_system_message:
        return npc.primary_directive

    system_message = f"""
.
..
...
....
.....
......
.......
........
.........
..........
Hello!
Welcome to the team.
You are the {npc.name} NPC with the following primary directive: {npc.primary_directive}.
Users may refer to you by your assistant name, {npc.name} and you should
consider this to be your core identity.
The current working directory is {os.getcwd()}.
The current date and time are : {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
"""

    if hasattr(npc, 'kg_data') and npc.kg_data:
        memory_context = npc.get_memory_context()
        if memory_context:
            system_message += f"\n\nMemory Context:\n{memory_context}\n"

    if npc.db_conn is not None:
        db_path = None
        if hasattr(npc.db_conn, "url") and npc.db_conn.url:
            db_path = npc.db_conn.url.database
        elif hasattr(npc.db_conn, "database"):
            db_path = npc.db_conn.database
        system_message += """What follows is information about the database connection. If you are asked to execute queries with tools, use this information. 
        If you are asked for help with debugging queries, use this information. 
        Do not unnecessarily reference that you possess this information unless it is
        specifically relevant to the request.

        DB Connection Information:        
        """
        if db_path:
            system_message += f"\nDatabase path: {db_path}\n"
        if npc.tables is not None:
            system_message += f"\nDatabase tables: {npc.tables}\n"

    if team is not None and npc.name == getattr(team, 'forenpc_name', 'sibiji'):
        team_context = team.context if hasattr(team, "context") and team.context else ""
        team_preferences = team.shared_context.get('preferences', '') if hasattr(team, "shared_context") else ""
        system_message += f"\nTeam context: {team_context}\n"
        if team_preferences:
            system_message += f"Team preferences: {team_preferences}\n"

        if hasattr(team, 'npcs') and team.npcs:
            members = []
            for name, member in team.npcs.items():
                if name != npc.name:
                    directive = getattr(member, 'primary_directive', '')
                    desc = directive[:50].strip() if directive else ''
                    members.append(f"  - @{name}: {desc}")
            if members:
                system_message += "\nTeam members available for delegation:\n" + "\n".join(members) + "\n"

    if hasattr(npc, 'jinxes_dict') and npc.jinxes_dict:
        tool_lines = []
        for jname, jinx in npc.jinxes_dict.items():
            desc = getattr(jinx, 'description', '') or ''
            tool_lines.append(f"  - {jname}: {desc.strip()}")
        if tool_lines:
            system_message += "\nYou have access to the following jinxes:\n"
            system_message += "\n".join(tool_lines) + "\n"
            if tool_capable:
                system_message += "\nUse the provided function calling interface to invoke tools when they are relevant to the request. For multi-step tasks, call one tool at a time and use its result to inform the next step.\n"
            else:
                jinx_names_str = ", ".join(npc.jinxes_dict.keys())
                jinx_instructions = f"""
                if you are in the [ReAct loop] and you are asked to use jinxes, refer to these guidelines:
              [BEGIN GUIDELINES FOR JINX EXECUTION]
                  Use jinxes when appropriate. For example:

                    - If you are asked about something up-to-date or dynamic (e.g., latest exchange rates)
                    - If the user asks you to read or edit a file
                    - If the user asks for code that should be executed
                    - If the user requests to open, search, download or scrape, which involve actual system or web actions
                    - If they request a screenshot, audio, or image manipulation
                    - Situations requiring file parsing (e.g., CSV or JSON loading)
                    - Scripted workflows or pipelines, e.g., generate a chart, fetch data, summarize from source, etc.

                    You MUST use a jinx if the request directly refers to a tool the AI cannot handle directly (e.g., 'run', 'open', 'search', etc).

                    You do not need to use a jinx if:

                    - the user asks a simple question like 'what is 2+2' or 'who invented linux', essentially any question which only requires general knowledge.
                    - The user asks you to write them a story (unless they separately specify saving it to a file, then you should directly write the story to be output through a jinx to said file.)
                    To invoke a jinx, return the action 'invoke_jinx' along with the jinx specific name.
                    An example for a jinx-specific return would be:
                    """ +"""
                    {
                        "action": "invoke_jinx",
                        "jinx_name": "file_reader",
                        "explanation": "Read the contents of <full_filename_path_from_user_request> and <detailed explanation of how to accomplish the problem outlined in the request>."
                    }


                    Do not use the jinx names as the action keys. You must use the action 'invoke_jinx' to invoke a jinx!
                    Do not invent jinx names. Use only those provided.


                Respond with a single JSON object only.
                To use a jinx, set action to jinx, jinx_name to one of [{jinx_names_str}], and inputs with the required parameters.

              [END GUIDELINES FOR JINX EXECUTION]


"""
                system_message += jinx_instructions




    return system_message

get_transformers_response(prompt=None, model=None, tokenizer=None, tools=None, tool_map=None, format=None, messages=None, auto_process_tool_calls=False, **kwargs)

Source code in npcpy/gen/response.py
def get_transformers_response(
   prompt: str = None,
   model=None,
   tokenizer=None, 
   tools: list = None,
   tool_map: Dict = None,
   format: str = None,
   messages: List[Dict[str, str]] = None,
   auto_process_tool_calls: bool = False,
   **kwargs,
) -> Dict[str, Any]:
   import torch
   import json
   import uuid
   from transformers import AutoTokenizer, AutoModelForCausalLM

   result = {
       "response": None,
       "messages": messages.copy() if messages else [],
       "raw_response": None,
       "tool_calls": [], 
       "tool_results": []
   }

   if model is None or tokenizer is None:
       model_name = model if isinstance(model, str) else "Qwen/Qwen3-1.7b"
       tokenizer = AutoTokenizer.from_pretrained(model_name)
       model = AutoModelForCausalLM.from_pretrained(model_name)

       if tokenizer.pad_token is None:
           tokenizer.pad_token = tokenizer.eos_token

   if prompt:
       if result['messages'] and result['messages'][-1]["role"] == "user":
           result['messages'][-1]["content"] = prompt
       else:
           result['messages'].append({"role": "user", "content": prompt})

   if format == "json":
       json_instruction = """If you are returning a json object, begin directly with the opening {.
Do not include any additional markdown formatting or leading ```json tags in your response."""
       if result["messages"] and result["messages"][-1]["role"] == "user":
           result["messages"][-1]["content"] += "\n" + json_instruction

   chat_text = tokenizer.apply_chat_template(result["messages"], tokenize=False, add_generation_prompt=True)
   device = next(model.parameters()).device
   inputs = tokenizer(chat_text, return_tensors="pt", padding=True, truncation=True)
   inputs = {k: v.to(device) for k, v in inputs.items()}


   with torch.no_grad():
       outputs = model.generate(
           **inputs,
           max_new_tokens=256,
           temperature=0.7,
           do_sample=True,
           pad_token_id=tokenizer.eos_token_id,
       )

   response_content = tokenizer.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True).strip()
   result["response"] = response_content
   result["raw_response"] = response_content
   result["messages"].append({"role": "assistant", "content": response_content})

   if auto_process_tool_calls and tools and tool_map:
       detected_tools = []
       for tool in tools:
           tool_name = tool.get("function", {}).get("name", "")
           if tool_name in response_content:
               detected_tools.append({
                   "id": str(uuid.uuid4()),
                   "function": {
                       "name": tool_name,
                       "arguments": "{}"
                   }
               })

       if detected_tools:
           result["tool_calls"] = detected_tools
           result = process_tool_calls(result, tool_map, "local", "transformers", result["messages"], tools=tools)

   if format == "json":
       try:
           if response_content.startswith("```json"):
               response_content = response_content.replace("```json", "").replace("```", "").strip()
           parsed_response = json.loads(response_content)
           result["response"] = parsed_response
       except json.JSONDecodeError:
           result["error"] = f"Invalid JSON response: {response_content}"

   return result

handle_streaming_json(api_params)

Handles streaming responses when JSON format is requested from LiteLLM.

Source code in npcpy/gen/response.py
def handle_streaming_json(api_params):
    """
    Handles streaming responses when JSON format is requested from LiteLLM.
    """
    json_buffer = ""
    stream = completion(**api_params)
    for chunk in stream:
        content = chunk.choices[0].delta.content
        if content:
            json_buffer += content
            try:
                json.loads(json_buffer)
                yield chunk
            except json.JSONDecodeError:
                pass

lookup_provider(model)

Determine the provider based on the model name. Checks custom providers first, then falls back to known providers.

Parameters:
  • model (str) –

    The model name

Returns:
  • str

    The provider name or None if not found

Source code in npcpy/npc_sysenv.py
def lookup_provider(model: str) -> str:
    """
    Determine the provider based on the model name.
    Checks custom providers first, then falls back to known providers.

    Args:
        model: The model name

    Returns:
        The provider name or None if not found
    """
    if model and os.path.isdir(os.path.expanduser(model)):
        adapter_config = os.path.join(os.path.expanduser(model), 'adapter_config.json')
        if os.path.exists(adapter_config):
            return "lora"

    custom_providers = load_custom_providers()

    for provider_name, config in custom_providers.items():
        if model.startswith(f"{provider_name}-"):
            return provider_name

        try:
            import requests
            api_key_var = config.get('api_key_var') or \
                         f"{provider_name.upper()}_API_KEY"
            api_key = os.environ.get(api_key_var)

            if api_key:
                base_url = config.get('base_url', '')
                headers = config.get('headers', {})
                headers['Authorization'] = f'Bearer {api_key}'

                models_endpoint = f"{base_url.rstrip('/')}/models"
                response = requests.get(
                    models_endpoint, 
                    headers=headers, 
                    timeout=1.0
                )

                if response.status_code == 200:
                    data = response.json()
                    models = []

                    if isinstance(data, dict) and 'data' in data:
                        models = [m['id'] for m in data['data']]
                    elif isinstance(data, list):
                        models = [m['id'] for m in data]

                    if model in models:
                        return provider_name
        except Exception:
            pass

    if model == "deepseek-chat" or model == "deepseek-reasoner":
        return "deepseek"

    if model.startswith("airllm-"):
        return "airllm"

    ollama_prefixes = [
        "llama", "deepseek", "qwen", "llava", 
        "phi", "mistral", "mixtral", "dolphin", 
        "codellama", "gemma",]
    if any(model.startswith(prefix) for prefix in ollama_prefixes):
        return "ollama"

    openai_prefixes = ["gpt-", "dall-e-", "whisper-", "o1"]
    if any(model.startswith(prefix) for prefix in openai_prefixes):
        return "openai"

    if model.startswith("claude"):
        return "anthropic"
    if model.startswith("gemini"):
        return "gemini"
    if "diffusion" in model:
        return "diffusers"

    return None

process_tool_calls(response_dict, tool_map, model, provider, messages, stream=False, tools=None)

Source code in npcpy/gen/response.py
def process_tool_calls(response_dict, tool_map, model, provider, messages, stream=False, tools=None):
    result = response_dict.copy()
    result["tool_results"] = []

    if "messages" not in result:
        result["messages"] = messages if messages else []

    tool_calls = result.get("tool_calls", [])

    if not tool_calls:
        return result

    tool_calls_for_message = []
    for tc in tool_calls:
        if isinstance(tc, dict):
            tool_calls_for_message.append(tc)
        else:
            tool_calls_for_message.append({
                "id": getattr(tc, "id", str(uuid.uuid4())),
                "type": "function",
                "function": {
                    "name": getattr(tc.function, "name", "") if hasattr(tc, "function") else "",
                    "arguments": getattr(tc.function, "arguments", "{}") if hasattr(tc, "function") else "{}"
                }
            })

    result["messages"].append({
        "role": "assistant",
        "content": None,
        "tool_calls": tool_calls_for_message
    })

    for tool_call in tool_calls:
        tool_id = str(uuid.uuid4())
        tool_name = None
        arguments = {}


        if isinstance(tool_call, dict):
            tool_id = tool_call.get("id", str(uuid.uuid4()))
            tool_name = tool_call.get("function", {}).get("name")
            arguments_str = tool_call.get("function", {}).get("arguments", "{}")
        else:
            tool_id = getattr(tool_call, "id", str(uuid.uuid4()))
            if hasattr(tool_call, "function"):
                func_obj = tool_call.function
                tool_name = getattr(func_obj, "name", None)
                arguments_str = getattr(func_obj, "arguments", "{}")
            else:
                continue

        try:
            arguments = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
        except json.JSONDecodeError:
            arguments = {"raw_arguments": arguments_str}


        if tool_name in tool_map:
            tool_result = None
            tool_result_str = ""
            serializable_result = None

            try:
                tool_result = tool_map[tool_name](**arguments)
            except Exception as e:
                tool_result = f"Error executing tool '{tool_name}': {str(e)}"

            try:
                tool_result_str = json.dumps(tool_result, default=str)
                try:
                    serializable_result = json.loads(tool_result_str)
                except json.JSONDecodeError:
                    serializable_result = {"result": tool_result_str}
            except Exception as e_serialize:
                tool_result_str = f"Error serializing result for {tool_name}: {str(e_serialize)}"
                serializable_result = {"error": tool_result_str}

            result["tool_results"].append({
                "tool_call_id": tool_id,
                "tool_name": tool_name,
                "arguments": arguments,
                "result": serializable_result
            })

            result["messages"].append({
                "role": "tool",
                "tool_call_id": tool_id,
                "content": tool_result_str
            })

    return result

render_markdown(text)

Renders markdown text, but handles code blocks as plain syntax-highlighted text.

Source code in npcpy/npc_sysenv.py
def render_markdown(text: str) -> None:
    """
    Renders markdown text, but handles code blocks as plain syntax-highlighted text.
    """
    lines = text.split("\n")
    console = Console()

    inside_code_block = False
    code_lines = []
    prose_lines = []
    lang = None

    def _flush_prose():
        if not prose_lines:
            return
        import re
        # Normalize CR/CRLF and collapse 3+ consecutive blank lines to 2.
        block = "\n".join(ln.rstrip('\r') for ln in prose_lines)
        block = re.sub(r'\n{3,}', '\n\n', block)

        _box_re = re.compile(r'[─-╿]')  # U+2500–U+257F box drawing block

        def _is_structured(ln: str) -> bool:
            """Box-drawing art or 4-space-indented code: must be rendered verbatim."""
            s = ln.strip()
            return bool(s) and (bool(_box_re.search(ln)) or ln.startswith('    '))

        lines = block.split('\n')
        segments: list = []   # (is_structured: bool, lines: list[str])
        struct_acc: list = []
        prose_acc: list = []

        def _commit_struct():
            if struct_acc:
                segments.append((True, list(struct_acc)))
                struct_acc.clear()

        def _commit_prose():
            if prose_acc:
                acc = list(prose_acc)
                while acc and not acc[0].strip():
                    acc.pop(0)
                while acc and not acc[-1].strip():
                    acc.pop()
                if acc:
                    segments.append((False, acc))
                prose_acc.clear()

        in_struct = False
        for i, ln in enumerate(lines):
            if _is_structured(ln):
                if not in_struct:
                    _commit_prose()
                    in_struct = True
                struct_acc.append(ln)
            elif not ln.strip():
                if in_struct:
                    # Suppress blank if the next non-blank line is also structured.
                    j = i + 1
                    while j < len(lines) and not lines[j].strip():
                        j += 1
                    if j < len(lines) and _is_structured(lines[j]):
                        pass  # drop blank between consecutive structured lines
                    else:
                        _commit_struct()
                        in_struct = False
                        prose_acc.append(ln)
                else:
                    prose_acc.append(ln)
            else:
                if in_struct:
                    _commit_struct()
                    in_struct = False
                prose_acc.append(ln)

        if in_struct:
            _commit_struct()
        else:
            _commit_prose()

        for is_str, seg_lines in segments:
            seg = '\n'.join(seg_lines)
            if not seg.strip():
                continue
            if is_str:
                # Verbatim output preserves box-drawing structure.
                # Rich Markdown would join consecutive lines as soft-breaks.
                sys.stdout.write(seg + '\n')
                sys.stdout.flush()
            else:
                console.print(Markdown(seg))

        prose_lines.clear()

    for line in lines:
        if line.startswith("```"):
            if inside_code_block:
                code = "\n".join(code_lines)
                if code.strip():
                    syntax = Syntax(
                        code, lang or "python", theme="monokai", line_numbers=False
                    )
                    console.print(syntax)
                code_lines = []
            else:
                _flush_prose()
                lang = line[3:].strip() or None
            inside_code_block = not inside_code_block
        elif inside_code_block:
            code_lines.append(line)
        else:
            prose_lines.append(line)

    _flush_prose()

sanitize_messages(messages)

Remove orphaned tool_use and tool_result blocks from message history.

Checks EVERY assistant message with tool_calls (not just the last one) to ensure Anthropic never sees a tool_use without a matching tool_result. For mid-history orphans, the tool_calls key is removed (keeping text content). For tail orphans, the assistant message is stripped entirely. Also merges consecutive same-role messages and ensures the conversation doesn't end with an assistant message (Anthropic rejects that).

Source code in npcpy/gen/response.py
def sanitize_messages(messages: list) -> list:
    """Remove orphaned tool_use and tool_result blocks from message history.

    Checks EVERY assistant message with tool_calls (not just the last one)
    to ensure Anthropic never sees a tool_use without a matching tool_result.
    For mid-history orphans, the tool_calls key is removed (keeping text content).
    For tail orphans, the assistant message is stripped entirely.
    Also merges consecutive same-role messages and ensures the conversation
    doesn't end with an assistant message (Anthropic rejects that).
    """
    if not messages:
        return messages

    def _extract_tc_ids(tool_calls_list):
        ids = set()
        for tc in tool_calls_list:
            if isinstance(tc, dict):
                tc_id = tc.get('id') or (tc.get('function') or {}).get('id', '')
            else:
                tc_id = getattr(tc, 'id', '')
            if tc_id:
                ids.add(tc_id)
        return ids

    # Pass 1: for each assistant with tool_calls, check that ALL tool_call_ids
    # are fulfilled by immediately following tool messages.  Collect fulfilled
    # IDs only from the contiguous run of tool messages right after the assistant.
    cleaned = []
    i = 0
    while i < len(messages):
        msg = messages[i]

        if msg.get('role') == 'assistant' and msg.get('tool_calls'):
            expected_ids = _extract_tc_ids(msg['tool_calls'])

            # Collect tool results that immediately follow this assistant message
            fulfilled_ids = set()
            j = i + 1
            while j < len(messages) and messages[j].get('role') == 'tool':
                tid = messages[j].get('tool_call_id', '')
                if tid:
                    fulfilled_ids.add(tid)
                j += 1

            if expected_ids and expected_ids.issubset(fulfilled_ids):
                # Valid pair — keep assistant and all its tool results
                cleaned.append(msg)
                for k in range(i + 1, j):
                    cleaned.append(messages[k])
                i = j
            elif not expected_ids and j > i + 1:
                # Tool calls without IDs (e.g. Ollama) but with following results — keep both
                cleaned.append(msg)
                for k in range(i + 1, j):
                    cleaned.append(messages[k])
                i = j
            else:
                # Orphaned tool_use — strip tool_calls, keep text content if any
                text_content = msg.get('content')
                if text_content:
                    cleaned.append({'role': 'assistant', 'content': text_content})
                # Also skip any partial tool results that followed
                i = j
        elif msg.get('role') == 'tool':
            # Stray tool result not preceded by an assistant with tool_calls
            content = msg.get('content', '')
            name = msg.get('name', 'tool')
            cleaned.append({
                'role': 'assistant',
                'content': f"[{name} result]: {content}" if name != 'tool' else content
            })
            i += 1
        else:
            cleaned.append(msg)
            i += 1

    # Pass 2: merge consecutive same-role messages (except system).
    # Anthropic requires strict user/assistant alternation.
    merged = []
    for msg in cleaned:
        role = msg.get('role', '')
        if (merged
                and role == merged[-1].get('role')
                and role in ('user', 'assistant')
                and not msg.get('tool_calls')
                and not merged[-1].get('tool_calls')):
            prev_content = merged[-1].get('content', '') or ''
            new_content = msg.get('content', '') or ''
            merged[-1]['content'] = (prev_content + '\n' + new_content).strip()
        else:
            merged.append(msg)

    # Pass 3: ensure conversation doesn't end with an assistant message.
    # Anthropic rejects "assistant message prefill" — last msg must be user/tool.
    while merged and merged[-1].get('role') == 'assistant' and not merged[-1].get('tool_calls'):
        # If the last non-system message is assistant text, drop it from the tail.
        # The caller will re-add the user prompt or tool result.
        merged.pop()

    return merged

embeddings

delete_embeddings_from_collection(collection, ids)

Delete embeddings by id from Chroma collection.

Source code in npcpy/gen/embeddings.py
def delete_embeddings_from_collection(collection, ids):
    """Delete embeddings by id from Chroma collection."""
    if ids:
        collection.delete(ids=ids)  

get_embeddings(texts, model, provider)

Generate embeddings using the specified provider and store them in Chroma.

Source code in npcpy/gen/embeddings.py
def get_embeddings(
    texts: List[str],
    model: str ,
    provider: str,
) -> List[List[float]]:
    """Generate embeddings using the specified provider and store them in Chroma."""
    if provider == "ollama":
        embeddings = get_ollama_embeddings(texts, model)
    elif provider == "openai":
        embeddings = get_openai_embeddings(texts, model)
    else:
        raise ValueError(f"Unsupported provider: {provider}")



    return embeddings

get_ollama_embeddings(texts, model='nomic-embed-text')

Generate embeddings using Ollama.

Source code in npcpy/gen/embeddings.py
def get_ollama_embeddings(
    texts: List[str], model: str = "nomic-embed-text"
) -> List[List[float]]:
    """Generate embeddings using Ollama."""
    import ollama

    embeddings = []
    for text in texts:
        response = ollama.embeddings(model=model, prompt=text)
        embeddings.append(response["embedding"])
    return embeddings

get_openai_embeddings(texts, model='text-embedding-3-small')

Generate embeddings using OpenAI.

Source code in npcpy/gen/embeddings.py
def get_openai_embeddings(
    texts: List[str], model: str = "text-embedding-3-small"
) -> List[List[float]]:
    """Generate embeddings using OpenAI."""
    client = OpenAI()
    response = client.embeddings.create(input=texts, model=model)
    return [embedding.embedding for embedding in response.data]

store_embeddings_for_model(texts, embeddings, chroma_client, model, provider, metadata=None)

Source code in npcpy/gen/embeddings.py
def store_embeddings_for_model(
    texts,
    embeddings,
    chroma_client,
    model,
    provider,
    metadata=None,
):
    collection_name = f"{provider}_{model}_embeddings"
    collection = chroma_client.get_collection(collection_name)


    if metadata is None:
        metadata = [{"text_length": len(text)} for text in texts]  
        print(
            "metadata is none, creating metadata for each document as the length of the text"
        )

    collection.add(
        ids=[str(i) for i in range(len(texts))],
        embeddings=embeddings,
        metadatas=metadata,  
        documents=texts,
    )