llama_cpp.py

from __future__ import annotations

import ctypes
import enum
import os
import pathlib

from ._ggml import (
    ggml_abort_callback,
    ggml_backend_sched_eval_callback,
    ggml_log_callback,
    ggml_opt_get_optimizer_params,
    ggml_cgraph,
    ggml_tensor_p,
    ggml_threadpool_p,
)

from typing import (
    Callable,
    Union,
    NewType,
    Optional,
    TYPE_CHECKING,
)

from llama_cpp._ctypes_extensions import (
    load_shared_library,
    byref,
    ctypes_function_for_shared_library,
)

if TYPE_CHECKING:
    from llama_cpp._ctypes_extensions import (
        CtypesCData,
        CtypesArray,
        CtypesPointer,
        CtypesVoidPointer,
        CtypesRef,
        CtypesPointerOrRef,
        CtypesFuncPointer,
    )


# Specify the base name of the shared library to load
_lib_base_name = "llama"
_override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH")
_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _override_base_path is None else pathlib.Path(_override_base_path)
# Load the library
_lib = load_shared_library(_lib_base_name, _base_path)

ctypes_function = ctypes_function_for_shared_library(_lib)


# llama.h bindings

_lib.llama_max_devices.argtypes = []
_lib.llama_max_devices.restype = ctypes.c_size_t

LLAMA_MAX_DEVICES = _lib.llama_max_devices()

LLAMA_MAX_SEQ = 256

# define LLAMA_DEFAULT_SEED 0xFFFFFFFF
LLAMA_DEFAULT_SEED = 0xFFFFFFFF

# define LLAMA_TOKEN_NULL -1
LLAMA_TOKEN_NULL = -1

# define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'
LLAMA_FILE_MAGIC_GGLA = 0x67676C61

# define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
LLAMA_FILE_MAGIC_GGSN = 0x6767736E

# define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'
LLAMA_FILE_MAGIC_GGSQ = 0x67677371

# define LLAMA_SESSION_MAGIC   LLAMA_FILE_MAGIC_GGSN
LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN
# define LLAMA_SESSION_VERSION 11
LLAMA_SESSION_VERSION = 11

# define LLAMA_STATE_SEQ_MAGIC   LLAMA_FILE_MAGIC_GGSQ
LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ
# define LLAMA_STATE_SEQ_VERSION 4
LLAMA_STATE_SEQ_VERSION = 4

# struct llama_vocab;
llama_vocab_p = NewType("llama_vocab_p", int)
llama_vocab_p_ctypes = ctypes.c_void_p

# struct llama_model;
llama_model_p = NewType("llama_model_p", int)
llama_model_p_ctypes = ctypes.c_void_p

# struct llama_context;
llama_context_p = NewType("llama_context_p", int)
llama_context_p_ctypes = ctypes.c_void_p

# struct llama_batch_ext;
llama_batch_ext_p = NewType("llama_batch_ext_p", int)
llama_batch_ext_p_ctypes = ctypes.c_void_p

# # struct llama_sampler;
# llama_sampler_p = NewType("llama_sampler_p", int)
# llama_sampler_p_ctypes = ctypes.c_void_p

# struct llama_opt_params;
llama_opt_params_p = NewType("llama_opt_params_p", int)
llama_opt_params_p_ctypes = ctypes.c_void_p

# typedef struct llama_memory_i * llama_memory_t;
llama_memory_i_p = NewType("llama_memory_i_p", int)
llama_memory_i_p_ctypes = ctypes.c_void_p

# typedef int32_t llama_pos;
llama_pos = ctypes.c_int32
# typedef int32_t llama_token;
llama_token = ctypes.c_int32
llama_token_p = ctypes.POINTER(llama_token)
# typedef int32_t llama_seq_id;
llama_seq_id = ctypes.c_int32


# enum llama_vocab_type {
#     LLAMA_VOCAB_TYPE_NONE   = 0, // For models without vocab
#     LLAMA_VOCAB_TYPE_SPM    = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
#     LLAMA_VOCAB_TYPE_BPE    = 2, // GPT-2 tokenizer based on byte-level BPE
#     LLAMA_VOCAB_TYPE_WPM    = 3, // BERT tokenizer based on WordPiece
#     LLAMA_VOCAB_TYPE_UGM    = 4, // T5 tokenizer based on Unigram
#     LLAMA_VOCAB_TYPE_RWKV   = 5, // RWKV tokenizer based on greedy tokenization
#     LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming
#     LLAMA_VOCAB_TYPE_TEST   = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex
# };
class LlamaVocabType(enum.IntEnum):
    """Vocabulary types; llama_vocab_type is the native query function."""

    LLAMA_VOCAB_TYPE_NONE   = 0
    """For models without vocab"""
    LLAMA_VOCAB_TYPE_SPM    = 1
    """LLaMA tokenizer based on byte-level BPE with byte fallback"""
    LLAMA_VOCAB_TYPE_BPE    = 2
    """GPT-2 tokenizer based on byte-level BPE"""
    LLAMA_VOCAB_TYPE_WPM    = 3
    """BERT tokenizer based on WordPiece"""
    LLAMA_VOCAB_TYPE_UGM    = 4
    """T5 tokenizer based on Unigram"""
    LLAMA_VOCAB_TYPE_RWKV   = 5
    """RWKV tokenizer based on greedy tokenization"""
    LLAMA_VOCAB_TYPE_PLAMO2 = 6
    """PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming"""
    LLAMA_VOCAB_TYPE_TEST   = 7
    """Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex"""


# Defined in llama.cpp/src/llama-vocab.h, not the public llama.h.
# // pre-tokenization types
# enum llama_vocab_pre_type {
#     LLAMA_VOCAB_PRE_TYPE_DEFAULT           = 0,
#     LLAMA_VOCAB_PRE_TYPE_LLAMA3            = 1,
#     LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM      = 2,
#     LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER    = 3,
#     LLAMA_VOCAB_PRE_TYPE_FALCON            = 4,
#     LLAMA_VOCAB_PRE_TYPE_MPT               = 5,
#     LLAMA_VOCAB_PRE_TYPE_STARCODER         = 6,
#     LLAMA_VOCAB_PRE_TYPE_GPT2              = 7,
#     LLAMA_VOCAB_PRE_TYPE_REFACT            = 8,
#     LLAMA_VOCAB_PRE_TYPE_COMMAND_R         = 9,
#     LLAMA_VOCAB_PRE_TYPE_STABLELM2         = 10,
#     LLAMA_VOCAB_PRE_TYPE_QWEN2             = 11,
#     LLAMA_VOCAB_PRE_TYPE_OLMO              = 12,
#     LLAMA_VOCAB_PRE_TYPE_DBRX              = 13,
#     LLAMA_VOCAB_PRE_TYPE_SMAUG             = 14,
#     LLAMA_VOCAB_PRE_TYPE_PORO              = 15,
#     LLAMA_VOCAB_PRE_TYPE_CHATGLM3          = 16,
#     LLAMA_VOCAB_PRE_TYPE_CHATGLM4          = 17,
#     LLAMA_VOCAB_PRE_TYPE_VIKING            = 18,
#     LLAMA_VOCAB_PRE_TYPE_JAIS              = 19,
#     LLAMA_VOCAB_PRE_TYPE_TEKKEN            = 20,
#     LLAMA_VOCAB_PRE_TYPE_SMOLLM            = 21,
#     LLAMA_VOCAB_PRE_TYPE_CODESHELL         = 22,
#     LLAMA_VOCAB_PRE_TYPE_BLOOM             = 23,
#     LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH      = 24,
#     LLAMA_VOCAB_PRE_TYPE_EXAONE            = 25,
#     LLAMA_VOCAB_PRE_TYPE_CHAMELEON         = 26,
#     LLAMA_VOCAB_PRE_TYPE_MINERVA           = 27,
#     LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM     = 28,
#     LLAMA_VOCAB_PRE_TYPE_GPT4O             = 29,
#     LLAMA_VOCAB_PRE_TYPE_SUPERBPE          = 30,
#     LLAMA_VOCAB_PRE_TYPE_TRILLION          = 31,
#     LLAMA_VOCAB_PRE_TYPE_BAILINGMOE        = 32,
#     LLAMA_VOCAB_PRE_TYPE_LLAMA4            = 33,
#     LLAMA_VOCAB_PRE_TYPE_PIXTRAL           = 34,
#     LLAMA_VOCAB_PRE_TYPE_SEED_CODER        = 35,
#     LLAMA_VOCAB_PRE_TYPE_HUNYUAN           = 36,
#     LLAMA_VOCAB_PRE_TYPE_KIMI_K2           = 37,
#     LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE     = 38,
#     LLAMA_VOCAB_PRE_TYPE_GROK_2            = 39,
#     LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING   = 40,
#     LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2        = 41,
#     LLAMA_VOCAB_PRE_TYPE_AFMOE             = 42,
#     LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN        = 43,
#     LLAMA_VOCAB_PRE_TYPE_YOUTU             = 44,
#     LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE        = 45,
#     LLAMA_VOCAB_PRE_TYPE_QWEN35            = 46,
#     LLAMA_VOCAB_PRE_TYPE_TINY_AYA          = 47,
#     LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM         = 48,
#     LLAMA_VOCAB_PRE_TYPE_JAIS2             = 49,
#     LLAMA_VOCAB_PRE_TYPE_GEMMA4            = 50,
#     LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE        = 51,
#     LLAMA_VOCAB_PRE_TYPE_MINICPM5          = 52,
#     LLAMA_VOCAB_PRE_TYPE_WHITESPACE        = 53,
#     LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54,
#     LLAMA_VOCAB_PRE_TYPE_MELLUM2           = 55,
#     LLAMA_VOCAB_PRE_TYPE_LAGUNA            = 56,
#     LLAMA_VOCAB_PRE_TYPE_HY_V4             = 57,
#     LLAMA_VOCAB_PRE_TYPE_SPARK2_5          = 58,
#     LLAMA_VOCAB_PRE_TYPE_UFAKZEKA          = 59,
#     LLAMA_VOCAB_PRE_TYPE_MMBERT            = 60,
#     LLAMA_VOCAB_PRE_TYPE_K2_HORIZON        = 61,
#     LLAMA_VOCAB_PRE_TYPE_FLINT             = 62,
#     LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL       = 63,
#     LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64,
# };
class llama_vocab_pre_type(enum.IntEnum):
    LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0
    LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1
    LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2
    LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3
    LLAMA_VOCAB_PRE_TYPE_FALCON = 4
    LLAMA_VOCAB_PRE_TYPE_MPT = 5
    LLAMA_VOCAB_PRE_TYPE_STARCODER = 6
    LLAMA_VOCAB_PRE_TYPE_GPT2 = 7
    LLAMA_VOCAB_PRE_TYPE_REFACT = 8
    LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9
    LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10
    LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11
    LLAMA_VOCAB_PRE_TYPE_OLMO = 12
    LLAMA_VOCAB_PRE_TYPE_DBRX = 13
    LLAMA_VOCAB_PRE_TYPE_SMAUG = 14
    LLAMA_VOCAB_PRE_TYPE_PORO = 15
    LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16
    LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17
    LLAMA_VOCAB_PRE_TYPE_VIKING = 18
    LLAMA_VOCAB_PRE_TYPE_JAIS = 19
    LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20
    LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21
    LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22
    LLAMA_VOCAB_PRE_TYPE_BLOOM = 23
    LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24
    LLAMA_VOCAB_PRE_TYPE_EXAONE = 25
    LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26
    LLAMA_VOCAB_PRE_TYPE_MINERVA = 27
    LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28
    LLAMA_VOCAB_PRE_TYPE_GPT4O = 29
    LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30
    LLAMA_VOCAB_PRE_TYPE_TRILLION = 31
    LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32
    LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33
    LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34
    LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35
    LLAMA_VOCAB_PRE_TYPE_HUNYUAN  = 36
    LLAMA_VOCAB_PRE_TYPE_KIMI_K2  = 37
    LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38
    LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39
    LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40
    LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41
    LLAMA_VOCAB_PRE_TYPE_AFMOE = 42
    LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43
    LLAMA_VOCAB_PRE_TYPE_YOUTU = 44
    LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45
    LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46
    LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47
    LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48
    LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49
    LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50
    LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51
    LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52
    LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53
    LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54
    LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55
    LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56
    LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57
    LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58
    LLAMA_VOCAB_PRE_TYPE_UFAKZEKA = 59
    LLAMA_VOCAB_PRE_TYPE_MMBERT = 60
    LLAMA_VOCAB_PRE_TYPE_K2_HORIZON = 61
    LLAMA_VOCAB_PRE_TYPE_FLINT = 62
    LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63
    LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64



# // note: these values should be synchronized with ggml_rope
# // TODO: maybe move this enum to ggml.h (ggml_rope_type)
# enum llama_rope_type {
#     LLAMA_ROPE_TYPE_NONE   = -1,
#     LLAMA_ROPE_TYPE_NORM   = 0,
#     LLAMA_ROPE_TYPE_NEOX   = GGML_ROPE_TYPE_NEOX,
#     LLAMA_ROPE_TYPE_MROPE  = GGML_ROPE_TYPE_MROPE,
#     LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,
#     LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,
# };
class llama_rope_type(enum.IntEnum):
    LLAMA_ROPE_TYPE_NONE = -1
    LLAMA_ROPE_TYPE_NORM = 0
    LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2
    LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8
    LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24
    LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE = 40


# enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
#     LLAMA_TOKEN_TYPE_UNDEFINED    = 0,
#     LLAMA_TOKEN_TYPE_NORMAL       = 1,
#     LLAMA_TOKEN_TYPE_UNKNOWN      = 2,
#     LLAMA_TOKEN_TYPE_CONTROL      = 3,
#     LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
#     LLAMA_TOKEN_TYPE_UNUSED       = 5,
#     LLAMA_TOKEN_TYPE_BYTE         = 6,
# };
class llama_token_type(enum.IntEnum):
    LLAMA_TOKEN_TYPE_UNDEFINED = 0
    LLAMA_TOKEN_TYPE_NORMAL = 1
    LLAMA_TOKEN_TYPE_UNKNOWN = 2
    LLAMA_TOKEN_TYPE_CONTROL = 3
    LLAMA_TOKEN_TYPE_USER_DEFINED = 4
    LLAMA_TOKEN_TYPE_UNUSED = 5
    LLAMA_TOKEN_TYPE_BYTE = 6


# enum llama_token_attr {
#     LLAMA_TOKEN_ATTR_UNDEFINED    = 0,
#     LLAMA_TOKEN_ATTR_UNKNOWN      = 1 << 0,
#     LLAMA_TOKEN_ATTR_UNUSED       = 1 << 1,
#     LLAMA_TOKEN_ATTR_NORMAL       = 1 << 2,
#     LLAMA_TOKEN_ATTR_CONTROL      = 1 << 3,  // SPECIAL?
#     LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
#     LLAMA_TOKEN_ATTR_BYTE         = 1 << 5,
#     LLAMA_TOKEN_ATTR_NORMALIZED   = 1 << 6,
#     LLAMA_TOKEN_ATTR_LSTRIP       = 1 << 7,
#     LLAMA_TOKEN_ATTR_RSTRIP       = 1 << 8,
#     LLAMA_TOKEN_ATTR_SINGLE_WORD  = 1 << 9,
# };
LLAMA_TOKEN_ATTR_UNDEFINED = 0
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0
LLAMA_TOKEN_ATTR_UNUSED = 1 << 1
LLAMA_TOKEN_ATTR_NORMAL = 1 << 2
LLAMA_TOKEN_ATTR_CONTROL = 1 << 3
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4
LLAMA_TOKEN_ATTR_BYTE = 1 << 5
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9


# // model file types
# enum llama_ftype {
#     LLAMA_FTYPE_ALL_F32              = 0,
#     LLAMA_FTYPE_MOSTLY_F16           = 1,  // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q4_0          = 2,  // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q4_1          = 3,  // except 1d tensors
#     // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4,  // tok_embeddings.weight and output.weight are F16
#     // LLAMA_FTYPE_MOSTLY_Q4_2       = 5,  // support has been removed
#     // LLAMA_FTYPE_MOSTLY_Q4_3       = 6,  // support has been removed
#     LLAMA_FTYPE_MOSTLY_Q8_0          = 7,  // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q5_0          = 8,  // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q5_1          = 9,  // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q2_K          = 10, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q3_K_S        = 11, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q3_K_M        = 12, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q3_K_L        = 13, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q4_K_S        = 14, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q4_K_M        = 15, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q5_K_S        = 16, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q5_K_M        = 17, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q6_K          = 18, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ2_XXS       = 19, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ2_XS        = 20, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q2_K_S        = 21, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ3_XS        = 22, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ3_XXS       = 23, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ1_S         = 24, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ4_NL        = 25, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ3_S         = 26, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ3_M         = 27, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ2_S         = 28, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ2_M         = 29, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ4_XS        = 30, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_IQ1_M         = 31, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_BF16          = 32, // except 1d tensors
#     //LLAMA_FTYPE_MOSTLY_Q4_0_4_4      = 33, // removed from gguf files, use Q4_0 and runtime repack
#     //LLAMA_FTYPE_MOSTLY_Q4_0_4_8      = 34, // removed from gguf files, use Q4_0 and runtime repack
#     //LLAMA_FTYPE_MOSTLY_Q4_0_8_8      = 35, // removed from gguf files, use Q4_0 and runtime repack
#     LLAMA_FTYPE_MOSTLY_TQ1_0         = 36, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_TQ2_0         = 37, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_MXFP4_MOE     = 38, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_NVFP4         = 39, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q1_0          = 40, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_Q2_0          = 41, // except 1d tensors
#     LLAMA_FTYPE_MOSTLY_F8_E4M3       = 42, // except 1d tensors
#
#     LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
# };
class llama_ftype(enum.IntEnum):
    LLAMA_FTYPE_ALL_F32 = 0
    LLAMA_FTYPE_MOSTLY_F16 = 1
    LLAMA_FTYPE_MOSTLY_Q4_0 = 2
    LLAMA_FTYPE_MOSTLY_Q4_1 = 3
    # LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4
    # LLAMA_FTYPE_MOSTLY_Q4_2 = 5
    # LLAMA_FTYPE_MOSTLY_Q4_3 = 6
    LLAMA_FTYPE_MOSTLY_Q8_0 = 7
    LLAMA_FTYPE_MOSTLY_Q5_0 = 8
    LLAMA_FTYPE_MOSTLY_Q5_1 = 9
    LLAMA_FTYPE_MOSTLY_Q2_K = 10
    LLAMA_FTYPE_MOSTLY_Q3_K_S = 11
    LLAMA_FTYPE_MOSTLY_Q3_K_M = 12
    LLAMA_FTYPE_MOSTLY_Q3_K_L = 13
    LLAMA_FTYPE_MOSTLY_Q4_K_S = 14
    LLAMA_FTYPE_MOSTLY_Q4_K_M = 15
    LLAMA_FTYPE_MOSTLY_Q5_K_S = 16
    LLAMA_FTYPE_MOSTLY_Q5_K_M = 17
    LLAMA_FTYPE_MOSTLY_Q6_K = 18
    LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19
    LLAMA_FTYPE_MOSTLY_IQ2_XS = 20
    LLAMA_FTYPE_MOSTLY_Q2_K_S = 21
    LLAMA_FTYPE_MOSTLY_IQ3_XS = 22
    LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23
    LLAMA_FTYPE_MOSTLY_IQ1_S = 24
    LLAMA_FTYPE_MOSTLY_IQ4_NL = 25
    LLAMA_FTYPE_MOSTLY_IQ3_S = 26
    LLAMA_FTYPE_MOSTLY_IQ3_M = 27
    LLAMA_FTYPE_MOSTLY_IQ2_S = 28
    LLAMA_FTYPE_MOSTLY_IQ2_M = 29
    LLAMA_FTYPE_MOSTLY_IQ4_XS = 30
    LLAMA_FTYPE_MOSTLY_IQ1_M = 31
    LLAMA_FTYPE_MOSTLY_BF16 = 32
    # LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33
    # LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34
    # LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35
    LLAMA_FTYPE_MOSTLY_TQ1_0 = 36
    LLAMA_FTYPE_MOSTLY_TQ2_0 = 37
    LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38
    LLAMA_FTYPE_MOSTLY_NVFP4 = 39
    LLAMA_FTYPE_MOSTLY_Q1_0 = 40
    LLAMA_FTYPE_MOSTLY_Q2_0 = 41
    LLAMA_FTYPE_MOSTLY_F8_E4M3 = 42
    LLAMA_FTYPE_GUESSED = 1024

# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"
# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);
@ctypes_function(
    "llama_ftype_name",
    [ctypes.c_int],
    ctypes.c_char_p,
)
def llama_ftype_name(
    ftype: llama_ftype, /
) -> bytes:
    """
    Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"
    """

# enum llama_rope_scaling_type {
#     LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,
#     LLAMA_ROPE_SCALING_TYPE_NONE        = 0,
#     LLAMA_ROPE_SCALING_TYPE_LINEAR      = 1,
#     LLAMA_ROPE_SCALING_TYPE_YARN        = 2,
#     LLAMA_ROPE_SCALING_TYPE_LONGROPE    = 3,
#     LLAMA_ROPE_SCALING_TYPE_MAX_VALUE   = LLAMA_ROPE_SCALING_TYPE_LONGROPE,
# };
class llama_rope_scaling_type(enum.IntEnum):
    LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1
    LLAMA_ROPE_SCALING_TYPE_NONE = 0
    LLAMA_ROPE_SCALING_TYPE_LINEAR = 1
    LLAMA_ROPE_SCALING_TYPE_YARN = 2
    LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3
    LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE

# enum llama_pooling_type {
#     LLAMA_POOLING_TYPE_UNSPECIFIED = -1,
#     LLAMA_POOLING_TYPE_NONE = 0,
#     LLAMA_POOLING_TYPE_MEAN = 1,
#     LLAMA_POOLING_TYPE_CLS  = 2,
#     LLAMA_POOLING_TYPE_LAST = 3,
#     LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph
# };
LLAMA_POOLING_TYPE_UNSPECIFIED = -1
LLAMA_POOLING_TYPE_NONE = 0
LLAMA_POOLING_TYPE_MEAN = 1
LLAMA_POOLING_TYPE_CLS = 2
LLAMA_POOLING_TYPE_LAST = 3
LLAMA_POOLING_TYPE_RANK = 4

# enum llama_attention_type {
#     LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,
#     LLAMA_ATTENTION_TYPE_CAUSAL      = 0,
#     LLAMA_ATTENTION_TYPE_NON_CAUSAL  = 1,
# };
class llama_attention_type(enum.IntEnum):
    LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1
    LLAMA_ATTENTION_TYPE_CAUSAL = 0
    LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1

# enum llama_flash_attn_type {
#     LLAMA_FLASH_ATTN_TYPE_AUTO     = -1,
#     LLAMA_FLASH_ATTN_TYPE_DISABLED = 0,
#     LLAMA_FLASH_ATTN_TYPE_ENABLED  = 1,
# };
class llama_flash_attn_type(enum.IntEnum):
    LLAMA_FLASH_ATTN_TYPE_AUTO     = -1
    LLAMA_FLASH_ATTN_TYPE_DISABLED = 0
    LLAMA_FLASH_ATTN_TYPE_ENABLED  = 1

# LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);
@ctypes_function(
    "llama_flash_attn_type_name",
    [ctypes.c_int],
    ctypes.c_char_p,
)
def llama_flash_attn_type_name(
    flash_attn_type: llama_flash_attn_type, /
) -> bytes:
    """
    Gets the name of a llama_flash_attn_type.
    """

# enum llama_split_mode {
#     LLAMA_SPLIT_MODE_NONE   = 0, // single GPU
#     LLAMA_SPLIT_MODE_LAYER  = 1, // split layers and KV across GPUs
#     LLAMA_SPLIT_MODE_ROW    = 2, // split layers and KV across GPUs, use tensor parallelism if supported
#     LLAMA_SPLIT_MODE_TENSOR = 3,
# };
class llama_split_mode(enum.IntEnum):
    LLAMA_SPLIT_MODE_NONE   = 0
    LLAMA_SPLIT_MODE_LAYER  = 1
    LLAMA_SPLIT_MODE_ROW    = 2
    LLAMA_SPLIT_MODE_TENSOR = 3

# enum llama_load_mode {
#     LLAMA_LOAD_MODE_AUTO       = -1, // auto-detect based on device capabilities
#     LLAMA_LOAD_MODE_NONE       =  0, // no special loading mode
#     LLAMA_LOAD_MODE_MMAP       =  1, // memory map the model
#     LLAMA_LOAD_MODE_MLOCK      =  2, // force system to keep model in RAM rather than swapping or compressing
#     LLAMA_LOAD_MODE_MMAP_MLOCK =  3, // mmap + force system to keep model in RAM rather than swapping or compressing
#     LLAMA_LOAD_MODE_DIRECT_IO  =  4, // use direct I/O if available
# };
class llama_load_mode(enum.IntEnum):
    LLAMA_LOAD_MODE_AUTO       = -1  # auto-detect based on device capabilities
    LLAMA_LOAD_MODE_NONE       =  0  # no special loading mode
    LLAMA_LOAD_MODE_MMAP       =  1  # memory map the model
    LLAMA_LOAD_MODE_MLOCK      =  2  # force system to keep model in RAM rather than swapping or compressing
    LLAMA_LOAD_MODE_MMAP_MLOCK =  3  # mmap + force system to keep model in RAM rather than swapping or compressing
    LLAMA_LOAD_MODE_DIRECT_IO  =  4  # use direct I/O if available

# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p)
def llama_load_mode_name(load_mode: int) -> bytes:
    ...

# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int)
def llama_load_mode_from_str(str: ctypes.c_char_p) -> int:
    ...

# enum llama_lazy_mode {
#     LLAMA_LAZY_MODE_OFF  = 0, // always read the whole tensor up front
#     LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
#     LLAMA_LAZY_MODE_ON   = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
# };
class llama_lazy_mode(enum.IntEnum):
    LLAMA_LAZY_MODE_OFF  = 0  # always read the whole tensor up front
    LLAMA_LAZY_MODE_AUTO = 1  # lazy only for marked tensors larger than 4 GiB (requires mmap)
    LLAMA_LAZY_MODE_ON   = 2  # read the rows of tensors marked by the arch on demand (requires mmap)

# enum llama_context_type {
#     LLAMA_CONTEXT_TYPE_DEFAULT = 0,
#     LLAMA_CONTEXT_TYPE_MTP     = 1,
# };
class llama_context_type(enum.IntEnum):
    LLAMA_CONTEXT_TYPE_DEFAULT = 0
    LLAMA_CONTEXT_TYPE_MTP     = 1

# enum llama_process_type {
#     LLAMA_PROCESS_TYPE_ENCODE,
#     LLAMA_PROCESS_TYPE_DECODE,
# };
class llama_process_type(enum.IntEnum):
    LLAMA_PROCESS_TYPE_ENCODE = 0
    LLAMA_PROCESS_TYPE_DECODE = 1

# typedef struct llama_token_data {
#     llama_token id; // token id
#     float logit;    // log-odds of the token
#     float p;        // probability of the token
# } llama_token_data;
class llama_token_data(ctypes.Structure):
    """Used to store token data

    Attributes:
        id (llama_token): token id
        logit (float): log-odds of the token
        p (float): probability of the token"""

    if TYPE_CHECKING:
        id: llama_token
        logit: float
        p: float

    _fields_ = [
        ("id", llama_token),
        ("logit", ctypes.c_float),
        ("p", ctypes.c_float),
    ]


llama_token_data_p = ctypes.POINTER(llama_token_data)


# typedef struct llama_token_data_array {
#     // TODO: consider SoA
#     // NOTE: this pointer can be modified by the samplers
#     llama_token_data * data;
#     size_t size;
#     int64_t selected; // this is the index in the data array (i.e. not the token id)
#     bool sorted;      // note: do not assume the data is sorted - always check this flag
# } llama_token_data_array;
class llama_token_data_array(ctypes.Structure):
    """Used to sample tokens given logits

    Attributes:
        data (ctypes.Array[llama_token_data]): token data
        size (int): size of the array
        selected (int): index in the data array (i.e. not the token id)
        sorted (bool): whether the array is sorted"""

    if TYPE_CHECKING:
        data: CtypesArray[llama_token_data]
        size: int
        selected: int
        sorted: bool

    _fields_ = [
        ("data", llama_token_data_p),
        ("size", ctypes.c_size_t),
        ("selected", ctypes.c_int64),
        ("sorted", ctypes.c_bool),
    ]


llama_token_data_array_p = ctypes.POINTER(llama_token_data_array)

# typedef bool (*llama_progress_callback)(float progress, void * user_data);
llama_progress_callback = ctypes.CFUNCTYPE(
    ctypes.c_bool, ctypes.c_float, ctypes.c_void_p
)


# // Input data for llama_encode/llama_decode
# // A llama_batch object can contain input about one or many sequences
# // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
# //
# // - token  : the token ids of the input (used when embd is NULL)
# // - embd   : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
# // - pos    : the positions of the respective token in the sequence
# //            (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)
# // - seq_id : the sequence to which the respective token belongs
# //            (if set to NULL, the sequence ID will be assumed to be 0)
# // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
# // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
# //            (if set to NULL:
# //               - if embeddings: all tokens are output
# //               - if not:        only the last token is output
# //            )
# //
# typedef struct llama_batch {
#     int32_t n_tokens;

#     llama_token  *  token;
#     float        *  embd;
#     llama_pos    *  pos;
#     int32_t      *  n_seq_id;
#     llama_seq_id ** seq_id;
#     int8_t       *  logits;   // TODO: rename this to "output"
# } llama_batch;
class llama_batch(ctypes.Structure):
    """Input data for llama_decode

    A llama_batch object can contain input about one or many sequences

    The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens

    Attributes:
        n_tokens (ctypes.c_int32): number of tokens
        token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL)
        embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
        pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence
        seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs
        logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output
    """

    if TYPE_CHECKING:
        n_tokens: ctypes.c_int32
        token: CtypesArray[llama_token]
        embd: CtypesArray[ctypes.c_float]
        pos: CtypesArray[CtypesArray[llama_pos]]
        n_seq_id: CtypesArray[ctypes.c_int]
        seq_id: CtypesArray[CtypesArray[llama_seq_id]]
        logits: CtypesArray[ctypes.c_int8]

    _fields_ = [
        ("n_tokens", ctypes.c_int32),
        ("token", ctypes.POINTER(llama_token)),
        ("embd", ctypes.POINTER(ctypes.c_float)),
        ("pos", ctypes.POINTER(llama_pos)),
        ("n_seq_id", ctypes.POINTER(ctypes.c_int32)),
        ("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))),
        ("logits", ctypes.POINTER(ctypes.c_int8)),
    ]


# enum llama_model_kv_override_type {
#     LLAMA_KV_OVERRIDE_TYPE_INT,
#     LLAMA_KV_OVERRIDE_TYPE_FLOAT,
#     LLAMA_KV_OVERRIDE_TYPE_BOOL,
#     LLAMA_KV_OVERRIDE_TYPE_STR,
# };
class LlamaModelKVOverrideType(enum.IntEnum):
    LLAMA_KV_OVERRIDE_TYPE_INT   = 0
    LLAMA_KV_OVERRIDE_TYPE_FLOAT = 1
    LLAMA_KV_OVERRIDE_TYPE_BOOL  = 2
    LLAMA_KV_OVERRIDE_TYPE_STR   = 3


# enum llama_model_meta_key {
#     LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,
#     LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,
#     LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,
#     LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,
#     LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,
#     LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,
#     LLAMA_MODEL_META_KEY_SAMPLING_TEMP,
#     LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,
#     LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,
#     LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,
#     LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,
#     LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,
# };
class LlamaModelMetaKey(enum.IntEnum):
    LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE        = 0
    LLAMA_MODEL_META_KEY_SAMPLING_TOP_K           = 1
    LLAMA_MODEL_META_KEY_SAMPLING_TOP_P           = 2
    LLAMA_MODEL_META_KEY_SAMPLING_MIN_P           = 3
    LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY = 4
    LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD   = 5
    LLAMA_MODEL_META_KEY_SAMPLING_TEMP            = 6
    LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N  = 7
    LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT  = 8
    LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT        = 9
    LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU    = 10
    LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA    = 11


# struct llama_model_kv_override {
#     enum llama_model_kv_override_type tag;

#     char key[128];


#     union {
#         int64_t val_i64;
#         double  val_f64;
#         bool    val_bool;
#         char    val_str[128];
#     };
# };
class llama_model_kv_override_value(ctypes.Union):
    _fields_ = [
        ("val_i64", ctypes.c_int64),
        ("val_f64", ctypes.c_double),
        ("val_bool", ctypes.c_bool),
        ("val_str", ctypes.c_char * 128),
    ]

    if TYPE_CHECKING:
        val_i64: int
        val_f64: float
        val_bool: bool
        val_str: bytes


class llama_model_kv_override(ctypes.Structure):
    _fields_ = [
        ("tag", ctypes.c_int),
        ("key", ctypes.c_char * 128),
        ("value", llama_model_kv_override_value),
    ]

    if TYPE_CHECKING:
        tag: int
        key: bytes
        value: Union[int, float, bool, bytes]

llama_model_kv_override_p = ctypes.POINTER(llama_model_kv_override)

# struct llama_model_tensor_buft_override {
#     const char * pattern;
#     ggml_backend_buffer_type_t buft;
# };
class llama_model_tensor_buft_override(ctypes.Structure):
    _fields_ = [
        ("pattern", ctypes.c_char_p),
        ("buft", ctypes.c_void_p),
    ]

    if TYPE_CHECKING:
        pattern: ctypes.c_char_p
        buft: ctypes.c_void_p

# struct llama_model_params {
#     // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
#     ggml_backend_dev_t * devices;

#     // NULL-terminated list of buffer types to use for tensors that match a pattern
#     const struct llama_model_tensor_buft_override * tensor_buft_overrides;

#     int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers
#     enum llama_split_mode split_mode; // how to split the model across multiple GPUs
#     enum llama_load_mode  load_mode;  // how to load the model

#     enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch

#     // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
#     int32_t main_gpu;

#     // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
#     const float * tensor_split;

#     // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
#     // If the provided progress_callback returns true, model loading continues.
#     // If it returns false, model loading is immediately aborted.
#     llama_progress_callback progress_callback;

#     // context pointer passed to the progress callback
#     void * progress_callback_user_data;

#     // override key-value pairs of the model meta data
#     const struct llama_model_kv_override * kv_overrides;

#     // Keep the booleans together to avoid misalignment during copy-by-value.
#     bool vocab_only;      // only load the vocabulary, no weights
#     bool check_tensors;   // validate model tensor data
#     bool use_extra_bufts; // use extra buffer types (used for weight repacking)
#     bool no_host;         // bypass host buffer allowing extra buffers to be used
#     bool no_alloc;        // only load metadata and simulate memory allocations
#     bool load_mtp;        // whether to load MTP layers
# };
class llama_model_params(ctypes.Structure):
    """Parameters for llama_model

    Attributes:
        devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
        tensor_buft_overrides(llama_model_tensor_buft_override): NULL-terminated list of buffer types to use for tensors that match a pattern
        n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers
        split_mode (int): how to split the model across multiple GPUs
        load_mode (int): how to load the model
        lazy_mode (int): on-demand reading of tensors marked by the arch
        main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored
        tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
        progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.
        progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback
        kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data
        vocab_only (bool): only load the vocabulary, no weights
        check_tensors (bool): validate model tensor data
        use_extra_bufts (bool): use extra buffer types (used for weight repacking)
        no_host (bool): bypass host buffer allowing extra buffers to be used
        no_alloc (bool): only load metadata and simulate memory allocations
        load_mtp (bool): whether to load MTP layers"""

    if TYPE_CHECKING:
        devices: CtypesArray[ctypes.c_void_p]  # NOTE: unused
        tensor_buft_overrides: CtypesPointer[llama_model_tensor_buft_override]
        n_gpu_layers: int
        split_mode: int
        load_mode: int
        lazy_mode: int
        main_gpu: int
        tensor_split: CtypesArray[ctypes.c_float]
        progress_callback: Callable[[float, ctypes.c_void_p], bool]
        progress_callback_user_data: ctypes.c_void_p
        kv_overrides: CtypesArray[llama_model_kv_override]
        vocab_only: bool
        check_tensors: bool
        use_extra_bufts: bool
        no_host: bool
        no_alloc: bool
        load_mtp: bool

    _fields_ = [
        ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused
        ("tensor_buft_overrides", ctypes.POINTER(llama_model_tensor_buft_override)),
        ("n_gpu_layers", ctypes.c_int32),
        ("split_mode", ctypes.c_int),
        ("load_mode", ctypes.c_int),
        ("lazy_mode", ctypes.c_int),
        ("main_gpu", ctypes.c_int32),
        ("tensor_split", ctypes.POINTER(ctypes.c_float)),
        ("progress_callback", llama_progress_callback),
        ("progress_callback_user_data", ctypes.c_void_p),
        ("kv_overrides", ctypes.POINTER(llama_model_kv_override)),
        ("vocab_only", ctypes.c_bool),
        ("check_tensors", ctypes.c_bool),
        ("use_extra_bufts", ctypes.c_bool),
        ("no_host", ctypes.c_bool),
        ("no_alloc", ctypes.c_bool),
        ("load_mtp", ctypes.c_bool),
    ]

llama_model_params_p = ctypes.POINTER(llama_model_params)


# struct llama_sampler_seq_config {
#     llama_seq_id           seq_id;
#     struct llama_sampler * sampler;
# };
class llama_sampler_seq_config(ctypes.Structure):
    if TYPE_CHECKING:
        seq_id: llama_seq_id
        sampler: ctypes.c_void_p
    _fields_ = [
        ("seq_id", llama_seq_id),
        ("sampler", ctypes.c_void_p),
    ]

llama_sampler_seq_config_p = ctypes.POINTER(llama_sampler_seq_config)


# // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations
# //       https://github.com/ggml-org/llama.cpp/pull/7544
# struct llama_context_params {
#     uint32_t n_ctx;                 // text context, 0 = from model
#     uint32_t n_batch;               // logical maximum batch size that can be submitted to llama_decode
#     uint32_t n_ubatch;              // physical maximum batch size
#     uint32_t n_seq_max;             // max number of sequences (i.e. distinct states for recurrent models)
#     uint32_t n_rs_seq;              // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
#     uint32_t n_outputs_max;         // max outputs in a ubatch (0 = n_batch)
#     uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max)
#     int32_t  n_threads;             // number of threads to use for generation
#     int32_t  n_threads_batch;       // number of threads to use for batch processing

#     enum llama_context_type      ctx_type;          // set the context type (e.g. MTP)
#     enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
#     enum llama_pooling_type      pooling_type;      // whether to pool (sum) embedding results by sequence id
#     enum llama_attention_type    attention_type;    // attention type to use for embeddings
#     enum llama_flash_attn_type   flash_attn_type;   // when to enable Flash Attention

#     // ref: https://github.com/ggml-org/llama.cpp/pull/2054
#     float    rope_freq_base;   // RoPE base frequency, 0 = from model
#     float    rope_freq_scale;  // RoPE frequency scaling factor, 0 = from model
#     float    yarn_ext_factor;  // YaRN extrapolation mix factor, negative = from model
#     float    yarn_attn_factor; // YaRN magnitude scaling factor
#     float    yarn_beta_fast;   // YaRN low correction dim
#     float    yarn_beta_slow;   // YaRN high correction dim
#     uint32_t yarn_orig_ctx;    // YaRN original context size
#     float    defrag_thold;     // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default)

#     ggml_backend_sched_eval_callback cb_eval;
#     void * cb_eval_user_data;

#     enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]
#     enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]

#     size_t moe_cache_size; // device cache bytes for host-memory MoE experts, 0 = disabled [EXPERIMENTAL]

#     // Abort callback
#     // if it returns true, execution of llama_decode() will be aborted
#     // currently works only with CPU execution
#     ggml_abort_callback abort_callback;
#     void *              abort_callback_data;

#     // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.
#     bool embeddings;  // if true, extract embeddings (together with logits, except for a pooling type other than NONE)
#     bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU
#     bool no_perf;     // measure performance timings
#     bool op_offload;  // offload host tensor operations to device
#     bool swa_full;    // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
#                         // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases
#                         //       ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573
#     bool kv_unified;  // use a unified buffer across the input sequences when computing the attention
#                         // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix
#                         // ref: https://github.com/ggml-org/llama.cpp/pull/14363

#     // [EXPERIMENTAL]
#     // backend sampler chain configuration (make sure the caller keeps the sampler chains alive)
#     // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init)
#     struct llama_sampler_seq_config * samplers;
#     size_t                            n_samplers;
#     // a source/target/parent context
#     // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts
#     struct llama_context * ctx_other;
# };
class llama_context_params(ctypes.Structure):
    """Parameters for llama_context

    Attributes:
        n_ctx (int): text context, 0 = from model
        n_batch (int): logical maximum batch size that can be submitted to llama_decode
        n_ubatch (int): physical maximum batch size
        n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models)
        n_rs_seq (int): number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
        n_outputs_max (int): max outputs in a ubatch (0 = n_batch)
        n_outputs_max_per_seq (int): max outputs per sequence (0 = n_outputs_max)
        n_threads (int): number of threads to use for generation
        n_threads_batch (int): number of threads to use for batch processing

        ctx_type (int): set the context type (e.g. MTP)
        rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type`
        pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)
        attention_type (int): attention type to use for embeddings
        flash_attn_type (int): when to enable Flash Attention

        rope_freq_base (float): RoPE base frequency, 0 = from model
        rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model
        yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model
        yarn_attn_factor (float): YaRN magnitude scaling factor
        yarn_beta_fast (float): YaRN low correction dim
        yarn_beta_slow (float): YaRN high correction dim
        yarn_orig_ctx (int): YaRN original context size
        defrag_thold (float): [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default)

        cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval
        cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval

        type_k (int): data type for K cache
        type_v (int): data type for V cache

        moe_cache_size (int): Device cache bytes for host-memory MoE experts, split among devices like layers; 0 disables it. Experimental.

        abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted
        abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback

        embeddings (bool): if true, extract embeddings (together with logits, except for a pooling type other than NONE)
        offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU
        no_perf (bool): whether to measure performance timings
        op_offload(bool): whether to offload host tensor operations to device
        swa_full(bool): whether to use full-size SWA cache
        kv_unified(bool): use a unified buffer across the input sequences when computing the attention

        samplers(llama_sampler_seq_config *): the samplers must be sampler chains (i.e. use llama_sampler_chain_init)
        n_samplers(size_t): numbers of sampler chains

        ctx_other(llama_context *): a source/target/parent context can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts
    """

    if TYPE_CHECKING:
        n_ctx: int
        n_batch: int
        n_ubatch: int
        n_seq_max: int
        n_rs_seq: int
        n_outputs_max: int
        n_outputs_max_per_seq: int
        n_threads: int
        n_threads_batch: int
        ctx_type: int
        rope_scaling_type: int
        pooling_type: int
        attention_type: int
        flash_attn_type: int
        rope_freq_base: float
        rope_freq_scale: float
        yarn_ext_factor: float
        yarn_attn_factor: float
        yarn_beta_fast: float
        yarn_beta_slow: float
        yarn_orig_ctx: int
        defrag_thold: float
        cb_eval: Callable[[ctypes.c_void_p, bool], bool]
        cb_eval_user_data: ctypes.c_void_p
        type_k: int
        type_v: int
        moe_cache_size: int
        abort_callback: Callable[[ctypes.c_void_p], bool]
        abort_callback_data: ctypes.c_void_p
        embeddings: bool
        offload_kqv: bool
        no_perf: bool
        op_offload:bool
        swa_full:bool
        kv_unified:bool
        samplers: ctypes.c_void_p
        n_samplers: int
        ctx_other: ctypes.c_void_p

    _fields_ = [
        ("n_ctx", ctypes.c_uint32),
        ("n_batch", ctypes.c_uint32),
        ("n_ubatch", ctypes.c_uint32),
        ("n_seq_max", ctypes.c_uint32),
        ("n_rs_seq", ctypes.c_uint32),
        ("n_outputs_max", ctypes.c_uint32),
        ("n_outputs_max_per_seq", ctypes.c_uint32),
        ("n_threads", ctypes.c_int32),
        ("n_threads_batch", ctypes.c_int32),
        ("ctx_type", ctypes.c_int),
        ("rope_scaling_type", ctypes.c_int),
        ("pooling_type", ctypes.c_int),
        ("attention_type", ctypes.c_int),
        ("flash_attn_type", ctypes.c_int),
        ("rope_freq_base", ctypes.c_float),
        ("rope_freq_scale", ctypes.c_float),
        ("yarn_ext_factor", ctypes.c_float),
        ("yarn_attn_factor", ctypes.c_float),
        ("yarn_beta_fast", ctypes.c_float),
        ("yarn_beta_slow", ctypes.c_float),
        ("yarn_orig_ctx", ctypes.c_uint32),
        ("defrag_thold", ctypes.c_float),
        ("cb_eval", ggml_backend_sched_eval_callback),
        ("cb_eval_user_data", ctypes.c_void_p),
        ("type_k", ctypes.c_int),
        ("type_v", ctypes.c_int),
        ("moe_cache_size", ctypes.c_size_t),
        ("abort_callback", ggml_abort_callback),
        ("abort_callback_data", ctypes.c_void_p),
        ("embeddings", ctypes.c_bool),
        ("offload_kqv", ctypes.c_bool),
        ("no_perf", ctypes.c_bool),
        ("op_offload", ctypes.c_bool),
        ("swa_full", ctypes.c_bool),
        ("kv_unified", ctypes.c_bool),
        ("samplers", llama_sampler_seq_config_p),
        ("n_samplers", ctypes.c_size_t),
        ("ctx_other", ctypes.c_void_p),
    ]

llama_context_params_p = ctypes.POINTER(llama_context_params)


# struct llama_model_tensor_override {
#     const char * pattern;
#     enum ggml_type type;
# };
class llama_model_tensor_override(ctypes.Structure):
    _fields_ = [
        ("pattern", ctypes.c_char_p),
        ("type", ctypes.c_int),
    ]

    if TYPE_CHECKING:
        pattern: ctypes.c_char_p
        type: ctypes.c_int

llama_model_tensor_override_p = ctypes.POINTER(llama_model_tensor_override)


# struct llama_model_imatrix_data {
#     const char * name;
#     const float * data;
#     size_t size;
# };
class llama_model_imatrix_data(ctypes.Structure):
    _fields_ = [
        ("name", ctypes.c_char_p),
        ("data", ctypes.POINTER(ctypes.c_float)),
        ("size", ctypes.c_size_t),
    ]

    if TYPE_CHECKING:
        name: ctypes.c_char_p
        data: ctypes.POINTER(ctypes.c_float) # type: ignore
        size: ctypes.c_size_t

llama_model_imatrix_data_p = ctypes.POINTER(llama_model_imatrix_data)


# // model quantization parameters
# typedef struct llama_model_quantize_params {
#     int32_t nthread;                                            // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
#     enum llama_ftype ftype;                                     // quantize to this llama_ftype
#     enum ggml_type output_tensor_type;                          // output tensor type
#     enum ggml_type token_embedding_type;                        // token embeddings tensor type
#     bool allow_requantize;                                      // allow quantizing non-f32/f16 tensors
#     bool quantize_output_tensor;                                // quantize output.weight
#     bool only_copy;                                             // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
#     bool pure;                                                  // quantize all tensors to the default type
#     bool keep_split;                                            // quantize to the same number of shards
#     bool dry_run;                                               // calculate and show the final quantization size without performing quantization
#     const struct llama_model_imatrix_data * imatrix;            // pointer to importance matrix data
#     const struct llama_model_kv_override * kv_overrides;        // pointer to kv overrides
#     const struct llama_model_tensor_override * tt_overrides;    // pointer to tensor overrides
#     const int32_t * prune_layers;                               // pointer to layer indices to prune
#     size_t max_buf_size;                                        // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB)
# } llama_model_quantize_params;
class llama_model_quantize_params(ctypes.Structure):
    """Parameters for llama_model_quantize

    Attributes:
        nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
        ftype (int): quantize to this llama_ftype
        output_tensor_type (int): output tensor type
        token_embedding_type (int): token embeddings tensor type
        allow_requantize (bool): allow quantizing non-f32/f16 tensors
        quantize_output_tensor (bool): quantize output.weight
        only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
        pure (bool): quantize all tensors to the default type
        keep_split (bool): quantize to the same number of shards
        dry_run (bool): calculate and show the final quantization size without performing quantization
        imatrix (POINTER(llama_model_imatrix_data)): Pointer to importance matrix data.
        kv_overrides (POINTER(llama_model_kv_override)): Pointer to KV overrides.
        tt_overrides (POINTER(llama_model_tensor_override)): Pointer to tensor overrides.
        prune_layers (POINTER(c_int32)): Pointer to layer indices to prune.
        max_buf_size (size_t): max bytes of tensor rows kept in memory at once, 0 = default (8 GiB)
    """

    if TYPE_CHECKING:
        nthread: int
        ftype: int
        output_tensor_type: int
        token_embedding_type: int
        allow_requantize: bool
        quantize_output_tensor: bool
        only_copy: bool
        pure: bool
        keep_split: bool
        dry_run: bool
        imatrix: ctypes.POINTER(llama_model_imatrix_data) # type: ignore
        kv_overrides: ctypes.POINTER(llama_model_kv_override) # type: ignore
        tensor_types: ctypes.POINTER(llama_model_tensor_override) # type: ignore
        prune_layers: ctypes.POINTER(ctypes.c_int32) # type: ignore
        max_buf_size: ctypes.c_size_t

    _fields_ = [
        ("nthread", ctypes.c_int32),
        ("ftype", ctypes.c_int),
        ("output_tensor_type", ctypes.c_int),
        ("token_embedding_type", ctypes.c_int),
        ("allow_requantize", ctypes.c_bool),
        ("quantize_output_tensor", ctypes.c_bool),
        ("only_copy", ctypes.c_bool),
        ("pure", ctypes.c_bool),
        ("keep_split", ctypes.c_bool),
        ("dry_run", ctypes.c_bool),
        ("imatrix", ctypes.POINTER(llama_model_imatrix_data)),
        ("kv_overrides", ctypes.POINTER(llama_model_kv_override)),
        ("tt_overrides", ctypes.POINTER(llama_model_tensor_override)),
        ("prune_layers", ctypes.POINTER(ctypes.c_int32)),
        ("max_buf_size", ctypes.c_size_t),
    ]


# typedef struct llama_logit_bias {
#     llama_token token;
#     float bias;
# } llama_logit_bias;
class llama_logit_bias(ctypes.Structure):
    """Used to store logit bias

    Attributes:
        token (llama_token): token id
        bias (float): bias"""

    if TYPE_CHECKING:
        token: llama_token
        bias: float

    _fields_ = [
        ("token", llama_token),
        ("bias", ctypes.c_float),
    ]


llama_logit_bias_p = ctypes.POINTER(llama_logit_bias)


# typedef struct llama_sampler_chain_params {
#     bool no_perf; // whether to measure performance timings
# } llama_sampler_chain_params;
class llama_sampler_chain_params(ctypes.Structure):
    """Parameters for llama_sampler_chain

    Attributes:
        no_perf (bool): whether to measure performance timings"""

    if TYPE_CHECKING:
        no_perf: bool

    _fields_ = [
        ("no_perf", ctypes.c_bool),
    ]


# // used in chat template
# typedef struct llama_chat_message {
#     const char * role;
#     const char * content;
# } llama_chat_message;
class llama_chat_message(ctypes.Structure):
    _fields_ = [
        ("role", ctypes.c_char_p),
        ("content", ctypes.c_char_p),
    ]


# // lora adapter
# struct llama_adapter_lora {
#     llama_model * model = nullptr;

#     // map tensor name to lora_a_b
#     std::unordered_map<std::string, llama_adapter_lora_weight> ab_map;

#     std::vector<ggml_context_ptr> ctxs;
#     std::vector<ggml_backend_buffer_ptr> bufs;

#     float alpha;

#     // gguf metadata
#     std::unordered_map<std::string, std::string> gguf_kv;

#     // activated lora (aLoRA)
#     std::vector<llama_token> alora_invocation_tokens;

#     explicit llama_adapter_lora(llama_model * model) : model(model) {}
#     ~llama_adapter_lora() = default;

#     llama_adapter_lora_weight * get_weight(ggml_tensor * w);

#     uint32_t get_n_nodes() const {
#         return ab_map.size() * 6u; // a, b, scale, add, 2 x mul_mat
#     }
# };
llama_adapter_lora_p = ctypes.c_void_p
llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p)

# // llama_adapter_cvec
# struct llama_adapter_cvec {
#     ggml_tensor * tensor_for(int il) const;

#     ggml_tensor * apply_to(ggml_context * ctx, ggml_tensor * cur, int  il) const;

#     bool apply(
#             const llama_model & model,
#             const float * data,
#             size_t len,
#             int32_t n_embd,
#             int32_t il_start,
#             int32_t il_end);

# private:
#     bool init(const llama_model & model);

#     int32_t layer_start = -1;
#     int32_t layer_end   = -1;

#     std::vector<ggml_context_ptr> ctxs;
#     std::vector<ggml_backend_buffer_ptr> bufs;

#     std::vector<ggml_tensor *> tensors; // per layer
# };
llama_adapter_cvec_p = ctypes.c_void_p
llama_adapter_cvec_p_ctypes = ctypes.POINTER(ctypes.c_void_p)


# LLAMA_API const char * llama_version(void);
@ctypes_function(
    "llama_version",
    [],
    ctypes.c_char_p,
)
def llama_version() -> bytes:
    """Get libllama version"""
    ...


# // Helpers for getting default parameters
# LLAMA_API struct llama_model_params          llama_model_default_params(void);
@ctypes_function(
    "llama_model_default_params",
    [],
    llama_model_params,
)
def llama_model_default_params() -> llama_model_params:
    """Get default parameters for llama_model"""
    ...


# LLAMA_API struct llama_context_params        llama_context_default_params(void);
@ctypes_function(
    "llama_context_default_params",
    [],
    llama_context_params,
)
def llama_context_default_params() -> llama_context_params:
    """Get default parameters for llama_context"""
    ...


# LLAMA_API struct llama_sampler_chain_params  llama_sampler_chain_default_params(void);
@ctypes_function(
    "llama_sampler_chain_default_params",
    [],
    llama_sampler_chain_params,
)
def llama_sampler_chain_default_params() -> llama_sampler_chain_params:
    """Get default parameters for llama_sampler_chain"""
    ...


# LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);
@ctypes_function(
    "llama_model_quantize_default_params",
    [],
    llama_model_quantize_params,
)
def llama_model_quantize_default_params() -> llama_model_quantize_params:
    """Get default parameters for llama_model_quantize"""
    ...


# // Initialize the llama + ggml backend
# // Call once at the start of the program
# LLAMA_API void llama_backend_init(void);
@ctypes_function(
    "llama_backend_init",
    [],
    None,
)
def llama_backend_init():
    """Initialize the llama + ggml backend
    If numa is true, use NUMA optimizations
    Call once at the start of the program"""
    ...


# // numa strategies
# enum ggml_numa_strategy {
#     GGML_NUMA_STRATEGY_DISABLED   = 0,
#     GGML_NUMA_STRATEGY_DISTRIBUTE = 1,
#     GGML_NUMA_STRATEGY_ISOLATE    = 2,
#     GGML_NUMA_STRATEGY_NUMACTL    = 3,
#     GGML_NUMA_STRATEGY_MIRROR     = 4,
#     GGML_NUMA_STRATEGY_COUNT
# };
GGML_NUMA_STRATEGY_DISABLED = 0
GGML_NUMA_STRATEGY_DISTRIBUTE = 1
GGML_NUMA_STRATEGY_ISOLATE = 2
GGML_NUMA_STRATEGY_NUMACTL = 3
GGML_NUMA_STRATEGY_MIRROR = 4
GGML_NUMA_STRATEGY_COUNT = 5


# // Call once at the end of the program - currently only used for MPI
# LLAMA_API void llama_backend_free(void);
@ctypes_function(
    "llama_backend_free",
    [],
    None,
)
def llama_backend_free():
    """Call once at the end of the program - currently only used for MPI"""
    ...


# // Optional: enable numa optimizations
# // TODO: deprecate and make part of llama_backend_init()
# LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);
@ctypes_function(
    "llama_numa_init",
    [ctypes.c_int],
    None,
)
def llama_numa_init(numa: int, /):
    ...


# // Optional: an auto threadpool gets created in ggml if not passed explicitly
# LLAMA_API void llama_attach_threadpool(
#         struct llama_context * ctx,
#            ggml_threadpool_t   threadpool,
#            ggml_threadpool_t   threadpool_batch);
@ctypes_function(
    "llama_attach_threadpool",
    [llama_context_p_ctypes, ggml_threadpool_p, ggml_threadpool_p],
    None,
)
def llama_attach_threadpool(
    ctx: llama_context_p,
    threadpool: ggml_threadpool_p,
    threadpool_batch: Optional[ggml_threadpool_p],
    /,
):
    """Attach externally owned generation and batch threadpools.

    A null ``threadpool_batch`` makes llama.cpp use ``threadpool`` for both.
    The context borrows the handles and does not free them.
    """
    ...


# LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);
@ctypes_function("llama_detach_threadpool", [llama_context_p_ctypes], None)
def llama_detach_threadpool(ctx: llama_context_p, /):
    """Detach externally owned threadpools from a context."""
    ...


# typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata);
llama_model_set_tensor_data_t = ctypes.CFUNCTYPE(
    None,
    ctypes.c_void_p,
    ctypes.c_void_p
)


# // Create a new model from GGUF metadata as well as a function to set the tensor data
# //   - tensors are created as GGML_TYPE_F32 by default,
# //     override by adding a tensor with the same name but a different name to the context
# LLAMA_API struct llama_model * llama_model_init_from_user(
#                 struct gguf_context * metadata,
#         llama_model_set_tensor_data_t   set_tensor_data,    // function to initialize tensor data with
#                                 void * set_tensor_data_ud, // userdata for function
#             struct llama_model_params   params);
@ctypes_function(
    "llama_model_init_from_user",
    [
        ctypes.c_void_p,
        llama_model_set_tensor_data_t,
        ctypes.c_void_p,
        llama_model_params
    ],
    llama_model_p_ctypes,
)
def llama_model_init_from_user(
    metadata: ctypes.c_void_p,
    set_tensor_data: llama_model_set_tensor_data_t, # type: ignore
    set_tensor_data_ud: ctypes.c_void_p,
    params: llama_model_params,
    /
) -> Optional[llama_model_p]:
    """
    Create a new model from GGUF metadata as well as a function to set the tensor data
      - tensors are created as GGML_TYPE_F32 by default,
        override by adding a tensor with the same name but a different name to the context
    """
    ...


# DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(
#                          const char * path_model,
#           struct llama_model_params   params),
#         "use llama_model_load_from_file instead");
@ctypes_function(
    "llama_load_model_from_file",
    [ctypes.c_char_p, llama_model_params],
    llama_model_p_ctypes,
)
def llama_load_model_from_file(
    path_model: bytes, params: llama_model_params, /
) -> Optional[llama_model_p]:
    ...


# // Load a model from a file
# // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf
# // If the split file name does not follow this pattern, use llama_model_load_from_splits
# LLAMA_API struct llama_model * llama_model_load_from_file(
#                          const char * path_model,
#           struct llama_model_params   params);
@ctypes_function(
    "llama_model_load_from_file",
    [ctypes.c_char_p, llama_model_params],
    llama_model_p_ctypes,
)
def llama_model_load_from_file(
    path_model: bytes, params: llama_model_params, /
) -> Optional[llama_model_p]:
    """
    Load a model from a file
    If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf
    If the split file name does not follow this pattern, use llama_model_load_from_splits
    """
    ...


# // Load a model from an open FILE pointer
# // The GGUF is read from the current position, so it can be embedded in a larger file
# // mmap needs the GGUF data section at a file offset to be aligned to the CPU tensor alignment (32 bytes)
# LLAMA_API struct llama_model * llama_model_load_from_file_ptr(
#                                 FILE * file,
#             struct llama_model_params   params);
@ctypes_function(
    "llama_model_load_from_file_ptr",
    [ctypes.c_void_p, llama_model_params],
    llama_model_p_ctypes,
)
def llama_model_load_from_file_ptr(
    file: ctypes.c_void_p, params: llama_model_params, /
) -> Optional[llama_model_p]:
    """
    Load a model from an open FILE pointer
    """
    ...


# // Load a model from multiple splits (support custom naming scheme)
# // The paths must be in the correct order
# LLAMA_API struct llama_model * llama_model_load_from_splits(
#                          const char ** paths,
#                              size_t    n_paths,
#           struct llama_model_params    params);
@ctypes_function(
    "llama_model_load_from_splits",
    [ctypes.POINTER(ctypes.c_char_p), ctypes.c_size_t, llama_model_params],
    llama_model_p_ctypes,
)
def llama_model_load_from_splits(
    paths: list[bytes], n_paths: int, params: llama_model_params, /
) -> Optional[llama_model_p]:
    """
    Load a model from multiple splits (support custom naming scheme)
    The paths must be in the correct order
    """
    ...


# LLAMA_API void llama_model_save_to_file(
#         const struct llama_model * model,
#                     const char * path_model);
@ctypes_function(
    "llama_model_save_to_file",
    [llama_model_p_ctypes, ctypes.c_char_p],
    None,
)
def llama_model_save_to_file(model: llama_model_p, path_model: bytes, /):
    ...


# DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),
#         "use llama_model_free instead");
@ctypes_function(
    "llama_free_model",
    [llama_model_p_ctypes],
    None,
)
def llama_free_model(model: llama_model_p, /):
    """
    DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),
         "use llama_model_free instead")
    """
    ...


# LLAMA_API void llama_model_free(struct llama_model * model);
@ctypes_function(
    "llama_model_free",
    [llama_model_p_ctypes],
    None,
)
def llama_model_free(model: llama_model_p, /):
    ...


# LLAMA_API struct llama_context * llama_init_from_model(
#                  struct llama_model * model,
#         struct llama_context_params   params);
@ctypes_function(
    "llama_init_from_model",
    [llama_model_p_ctypes, llama_context_params],
    llama_context_p_ctypes,
)
def llama_init_from_model(
    model: llama_model_p, params: llama_context_params, /
) -> Optional[llama_context_p]:
    ...


# DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(
#                  struct llama_model * model,
#         struct llama_context_params   params),
#         "use llama_init_from_model instead");
@ctypes_function(
    "llama_new_context_with_model",
    [llama_model_p_ctypes, llama_context_params],
    llama_context_p_ctypes,
)
def llama_new_context_with_model(
    model: llama_model_p, params: llama_context_params, /
) -> Optional[llama_context_p]:
    ...


# // Frees all allocated memory
# LLAMA_API void llama_free(struct llama_context * ctx);
@ctypes_function(
    "llama_free",
    [llama_context_p_ctypes],
    None,
)
def llama_free(ctx: llama_context_p, /):
    """Frees all allocated memory"""
    ...


# enum common_params_fit_status { // llama.cpp/common/fit.h
#     COMMON_PARAMS_FIT_STATUS_SUCCESS = 0,
#     COMMON_PARAMS_FIT_STATUS_FAILURE = 1,
#     COMMON_PARAMS_FIT_STATUS_ERROR   = 2,
# };
class common_params_fit_status(enum.IntEnum):
    COMMON_PARAMS_FIT_STATUS_SUCCESS = 0
    COMMON_PARAMS_FIT_STATUS_FAILURE = 1
    COMMON_PARAMS_FIT_STATUS_ERROR = 2


class llama_params_fit_status(enum.IntEnum):
    """Compatibility names for the status now defined in common/fit.h."""

    LLAMA_PARAMS_FIT_STATUS_SUCCESS = common_params_fit_status.COMMON_PARAMS_FIT_STATUS_SUCCESS
    LLAMA_PARAMS_FIT_STATUS_FAILURE = common_params_fit_status.COMMON_PARAMS_FIT_STATUS_FAILURE
    LLAMA_PARAMS_FIT_STATUS_ERROR = common_params_fit_status.COMMON_PARAMS_FIT_STATUS_ERROR


# LLAMA_API int64_t llama_time_us(void);
@ctypes_function(
    "llama_time_us",
    [],
    ctypes.c_int64,
)
def llama_time_us() -> int:
    ...


# LLAMA_API size_t llama_max_devices(void);
@ctypes_function("llama_max_devices", [], ctypes.c_size_t)
def llama_max_devices() -> int:
    ...


# LLAMA_API size_t llama_max_parallel_sequences(void);
@ctypes_function("llama_max_parallel_sequences", [], ctypes.c_size_t)
def llama_max_parallel_sequences() -> int:
    ...


# LLAMA_API size_t llama_max_tensor_buft_overrides(void);
@ctypes_function("llama_max_tensor_buft_overrides", [], ctypes.c_size_t)
def llama_max_tensor_buft_overrides() -> int:
    ...


# LLAMA_API bool llama_supports_mmap       (void);
@ctypes_function("llama_supports_mmap", [], ctypes.c_bool)
def llama_supports_mmap() -> bool:
    ...


# LLAMA_API bool llama_supports_mlock      (void);
@ctypes_function("llama_supports_mlock", [], ctypes.c_bool)
def llama_supports_mlock() -> bool:
    ...


# LLAMA_API bool llama_supports_gpu_offload(void);
@ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool)
def llama_supports_gpu_offload() -> bool:
    ...


# LLAMA_API bool llama_supports_rpc        (void);
@ctypes_function("llama_supports_rpc", [], ctypes.c_bool)
def llama_supports_rpc() -> bool:
    ...

# // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions
# //       In some cases the requested values via llama_context_params may differ from the actual values used by the context
# //       ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
# LLAMA_API uint32_t llama_n_ctx      (const struct llama_context * ctx);
@ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ctx(ctx: llama_context_p, /) -> int:
    ...

# LLAMA_API uint32_t llama_n_ctx_seq  (const struct llama_context * ctx);
@ctypes_function("llama_n_ctx_seq", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ctx_seq(ctx: llama_context_p, /) -> int:
    ...


# LLAMA_API uint32_t llama_n_batch    (const struct llama_context * ctx);
@ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_batch(ctx: llama_context_p, /) -> int:
    ...


# LLAMA_API uint32_t llama_n_ubatch   (const struct llama_context * ctx);
@ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_ubatch(ctx: llama_context_p, /) -> int:
    ...


# LLAMA_API uint32_t llama_n_seq_max  (const struct llama_context * ctx);
@ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_seq_max(ctx: llama_context_p, /) -> int:
    ...


# LLAMA_API uint32_t llama_n_rs_seq   (const struct llama_context * ctx);
@ctypes_function("llama_n_rs_seq", [llama_context_p_ctypes], ctypes.c_uint32)
def llama_n_rs_seq(ctx: llama_context_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");
@ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_ctx_train(model: llama_model_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API int32_t llama_n_embd     (const struct llama_model * model), "use llama_model_n_embd instead");
@ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_embd(model: llama_model_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API int32_t llama_n_layer    (const struct llama_model * model), "use llama_model_n_layer instead");
@ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_layer(model: llama_model_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API int32_t llama_n_head     (const struct llama_model * model), "use llama_model_n_head instead");
@ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32)
def llama_n_head(model: llama_model_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API int32_t llama_n_vocab    (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");
@ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32)
def llama_n_vocab(model: llama_vocab_p, /) -> int:
    ...


# LLAMA_API const struct llama_model * llama_get_model   (const struct llama_context * ctx);
@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes)
def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]:
    ...


# LLAMA_API           llama_memory_t   llama_get_memory  (const struct llama_context * ctx);
@ctypes_function("llama_get_memory", [llama_context_p_ctypes], llama_memory_i_p_ctypes)
def llama_get_memory(ctx: llama_context_p, /) -> Optional[llama_memory_i_p]:
    ...

# LLAMA_API enum llama_pooling_type    llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int)
def llama_pooling_type(ctx: llama_context_p, /) -> int:
    ...


# LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
@ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes)
def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]:
    ...


# LLAMA_API enum llama_rope_type       llama_model_rope_type(const struct llama_model * model);
@ctypes_function("llama_model_rope_type", [llama_model_p_ctypes], ctypes.c_int)
def llama_model_rope_type(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);
@ctypes_function("llama_model_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_ctx_train(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_embd     (const struct llama_model * model);
@ctypes_function("llama_model_n_embd", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);
@ctypes_function("llama_model_n_embd_inp", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd_inp(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);
@ctypes_function("llama_model_n_embd_out", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_embd_out(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_layer    (const struct llama_model * model);
@ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_layer(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_layer_nextn(const struct llama_model * model);
@ctypes_function("llama_model_n_layer_nextn", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_layer_nextn(model: llama_model_p, /) -> int:
    ...

# LLAMA_API int32_t llama_model_n_head     (const struct llama_model * model);
@ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_head(model: llama_model_p, /) -> int:
    ...


# LLAMA_API int32_t llama_model_n_head_kv  (const struct llama_model * model);
@ctypes_function("llama_model_n_head_kv", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_head_kv(model: llama_model_p, /) -> int:
    ...


 # LLAMA_API int32_t llama_model_n_swa      (const struct llama_model * model);
@ctypes_function("llama_model_n_swa", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_n_swa(model: llama_model_p, /) -> int:
    ...


# // Get the model's RoPE frequency scaling factor
# LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);
@ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float)
def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float:
    """
    Get the model's RoPE frequency scaling factor
    """
    ...


# // Returns the number of classifier outputs (only valid for classifier models)
# // Undefined behavior for non-classifier models
# LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);
@ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32)
def llama_model_n_cls_out(model: llama_model_p, /) -> int:
    """
    Returns the number of classifier outputs (only valid for classifier models)
    Undefined behavior for non-classifier models
    """
    ...


# // Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided
# LLAMA_API const char * llama_model_cls_label(const struct llama_model * model, uint32_t i);
@ctypes_function("llama_model_cls_label", [llama_model_p_ctypes, ctypes.c_uint32], ctypes.c_char_p)
def llama_model_cls_label(model: llama_model_p, i: ctypes.c_uint32, /) -> ctypes.c_char_p:
    """
    Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided
    """
    ...


# LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab);
@ctypes_function("llama_vocab_type", [llama_vocab_p_ctypes], ctypes.c_int)
def llama_vocab_type(vocab: llama_vocab_p, /) -> int:
    ...


# LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);
@ctypes_function("llama_vocab_n_tokens", [llama_vocab_p_ctypes], ctypes.c_int32)
def llama_vocab_n_tokens(vocab: llama_vocab_p, /) -> int:
    ...


# // Functions to access the model's GGUF metadata scalar values
# // - The functions return the length of the string on success, or -1 on failure
# // - The output string is always null-terminated and cleared on failure
# // - When retrieving a string, an extra byte must be allocated to account for the null terminator
# // - GGUF array values are not supported by these functions


# // Get metadata value as a string by key name
# LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);
@ctypes_function(
    "llama_model_meta_val_str",
    [
        llama_model_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_model_meta_val_str(
    model: llama_model_p,
    key: Union[ctypes.c_char_p, bytes],
    buf: bytes,
    buf_size: int,
    /,
) -> int:
    """Get metadata value as a string by key name"""
    ...


# // Get the number of metadata key/value pairs
# LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);
@ctypes_function("llama_model_meta_count", [llama_model_p_ctypes], ctypes.c_int32)
def llama_model_meta_count(model: llama_model_p, /) -> int:
    """Get the number of metadata key/value pairs"""
    ...


# // Get sampling metadata key name. Returns nullptr if the key is invalid
# LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);
@ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p)
def llama_model_meta_key_str(key: int, /) -> ctypes.c_char_p:
    """Get sampling metadata key name. Returns nullptr if the key is invalid"""
    ...


# // Get metadata key name by index
# LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
@ctypes_function(
    "llama_model_meta_key_by_index",
    [
        llama_model_p_ctypes,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_model_meta_key_by_index(
    model: llama_model_p,
    i: Union[ctypes.c_int, int],
    buf: Union[bytes, CtypesArray[ctypes.c_char]],
    buf_size: int,
    /,
) -> int:
    """Get metadata key name by index"""
    ...


# // Get metadata value as a string by index
# LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
@ctypes_function(
    "llama_model_meta_val_str_by_index",
    [
        llama_model_p_ctypes,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_model_meta_val_str_by_index(
    model: llama_model_p,
    i: Union[ctypes.c_int, int],
    buf: Union[bytes, CtypesArray[ctypes.c_char]],
    buf_size: int,
    /,
) -> int:
    """Get metadata value as a string by index"""
    ...


# // Get a string describing the model type
# LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
@ctypes_function(
    "llama_model_desc",
    [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_size_t],
    ctypes.c_int32,
)
def llama_model_desc(
    model: llama_model_p,
    buf: Union[bytes, CtypesArray[ctypes.c_char]],
    buf_size: Union[ctypes.c_size_t, int],
    /,
) -> int:
    """Get a string describing the model type"""
    ...


# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0
# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model);
@ctypes_function(
    "llama_model_ftype",
    [llama_model_p_ctypes],
    ctypes.c_int,
)
def llama_model_ftype(
    model: llama_model_p,
    /,
) -> int:
    """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0"""
    ...


# // Returns the total size of all the tensors in the model in bytes
# LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
@ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64)
def llama_model_size(model: llama_model_p, /) -> int:
    """Returns the total size of all the tensors in the model in bytes"""
    ...


# // Get the default chat template. Returns nullptr if not available
# // If name is NULL, returns the default chat template
# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);
@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_char_p)
def llama_model_chat_template(model: llama_model_p, name: Optional[bytes], /) -> Optional[bytes]:
    """Get the default chat template. Returns None if not available
    If name is None, returns the default chat template"""
    ...


# // Returns the total number of parameters in the model
# LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
@ctypes_function("llama_model_n_params", [llama_model_p_ctypes], ctypes.c_uint64)
def llama_model_n_params(model: llama_model_p, /) -> int:
    """Returns the total number of parameters in the model"""
    ...


# // Returns true if the model contains an encoder that requires llama_encode() call
# LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);
@ctypes_function("llama_model_has_encoder", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_has_encoder(model: llama_model_p, /) -> bool:
    """Returns true if the model contains an encoder that requires llama_encode() call"""
    ...


# // Returns true if the model contains a decoder that requires llama_decode() call
# LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);
@ctypes_function("llama_model_has_decoder", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_has_decoder(model: llama_model_p, /) -> bool:
    """Returns true if the model contains a decoder that requires llama_decode() call"""
    ...


# // For encoder-decoder models, this function returns id of the token that must be provided
# // to the decoder to start generating output sequence. For other models, it returns -1.
# LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);
@ctypes_function(
    "llama_model_decoder_start_token", [llama_model_p_ctypes], llama_token
)
def llama_model_decoder_start_token(model: llama_model_p, /) -> ctypes.c_int32:
    """For encoder-decoder models, this function returns id of the token that must be provided
    to the decoder to start generating output sequence. For other models, it returns -1.
    """
    ...


# // Returns true if the model is recurrent (like Mamba, RWKV, etc.)
# LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);
@ctypes_function("llama_model_is_recurrent", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_is_recurrent(model: llama_model_p, /) -> bool:
    """Returns true if the model is recurrent (like Mamba, RWKV, etc.)"""
    ...


# // Returns true if the model is hybrid (like Jamba, Granite, etc.)
# LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model);
@ctypes_function("llama_model_is_hybrid", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_is_hybrid(model: llama_model_p, /) -> bool:
    """Returns true if the model is hybrid (like Jamba, Granite, etc.)"""
    ...


# // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)
# LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model);
@ctypes_function("llama_model_is_diffusion", [llama_model_p_ctypes], ctypes.c_bool)
def llama_model_is_diffusion(model: llama_model_p, /) -> bool:
    """Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)"""
    ...


# // Returns 0 on success
# LLAMA_API uint32_t llama_model_quantize(
#         const char * fname_inp,
#         const char * fname_out,
#         const llama_model_quantize_params * params);
@ctypes_function(
    "llama_model_quantize",
    [
        ctypes.c_char_p,
        ctypes.c_char_p,
        ctypes.POINTER(llama_model_quantize_params),
    ],
    ctypes.c_uint32,
)
def llama_model_quantize(
    fname_inp: bytes,
    fname_out: bytes,
    params: CtypesPointerOrRef[llama_model_quantize_params],
    /,
) -> int:
    """Returns 0 on success"""
    ...


# //
# // Adapters
# //

# // Load a LoRA adapter from file
# // The adapter is valid as long as the associated model is not freed
# LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(
#         struct llama_model * model,
#         const char * path_lora);
@ctypes_function(
    "llama_adapter_lora_init",
    [llama_model_p_ctypes, ctypes.c_char_p],
    llama_adapter_lora_p_ctypes,
)
def llama_adapter_lora_init(
    model: llama_model_p, path_lora: bytes, /
) -> Optional[llama_adapter_lora_p]:
    ...


# // Load a LoRA adapter from an open FILE pointer, reading from its current position
# LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init_from_file_ptr(
#         struct llama_model * model,
#         FILE * file);
@ctypes_function(
    "llama_adapter_lora_init_from_file_ptr",
    [llama_model_p_ctypes, ctypes.c_void_p],
    llama_adapter_lora_p_ctypes,
)
def llama_adapter_lora_init_from_file_ptr(
    model: llama_model_p, file: ctypes.c_void_p, /
) -> Optional[llama_adapter_lora_p]:
    ...


# // Functions to access the adapter's GGUF metadata scalar values
# // - The functions return the length of the string on success, or -1 on failure
# // - The output string is always null-terminated and cleared on failure
# // - When retrieving a string, an extra byte must be allocated to account for the null terminator
# // - GGUF array values are not supported by these functions

# // Get metadata value as a string by key name
# LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size);
@ctypes_function(
    "llama_adapter_meta_val_str",
    [
        llama_adapter_lora_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_adapter_meta_val_str(
    adapter: llama_adapter_lora_p,
    key: ctypes.c_char_p,
    buf: ctypes.c_char_p,
    buf_size: ctypes.c_size_t,
    /,
) -> int:
    """Get metadata value as a string by key name"""
    ...


# // Get the number of metadata key/value pairs
# LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter);
@ctypes_function(
    "llama_adapter_meta_count",
    [llama_adapter_lora_p_ctypes],
    ctypes.c_int32,
)
def llama_adapter_meta_count(adapter: llama_adapter_lora_p) -> int:
    """Get the number of metadata key/value pairs"""
    ...


# // Get metadata key name by index
# LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
@ctypes_function(
    "llama_adapter_meta_key_by_index",
    [
        llama_adapter_lora_p_ctypes,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_adapter_meta_key_by_index(
    adapter: llama_adapter_lora_p,
    i: ctypes.c_int32,
    buf: ctypes.c_char_p,
    buf_size: ctypes.c_size_t,
    /,
) -> int:
    """Get metadata key name by index"""
    ...


# // Get metadata value as a string by index
# LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
@ctypes_function(
    "llama_adapter_meta_val_str_by_index",
    [
        llama_adapter_lora_p_ctypes,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_adapter_meta_val_str_by_index(
    adapter: llama_adapter_lora_p,
    i: ctypes.c_int32,
    buf: ctypes.c_char_p,
    buf_size: ctypes.c_size_t,
    /,
) -> int:
    """Get metadata value as a string by index"""
    ...


# // Manually free a LoRA adapter
# // NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted
# LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);
@ctypes_function(
    "llama_adapter_lora_free",
    [llama_adapter_lora_p_ctypes],
    None,
)
def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /):
    ...


# // Get the invocation tokens if the current lora is an alora
# LLAMA_API uint64_t            llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);
@ctypes_function(
    "llama_adapter_get_alora_n_invocation_tokens",
    [llama_adapter_lora_p_ctypes],
    ctypes.c_uint64,
)
def llama_adapter_get_alora_n_invocation_tokens(adapter: llama_adapter_lora_p, /) -> ctypes.c_uint64:
    ...


# LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens  (const struct llama_adapter_lora * adapter);
@ctypes_function(
    "llama_adapter_get_alora_invocation_tokens",
    [llama_adapter_lora_p_ctypes],
    ctypes.POINTER(llama_token),
)
def llama_adapter_get_alora_invocation_tokens(adapter: llama_adapter_lora_p, /) -> CtypesPointer[llama_token]:
    ...


# // The following functions operate on a llama_context, hence the naming: llama_verb_...


# // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.
# LLAMA_API int32_t llama_set_adapters_lora(
#         struct llama_context * ctx,
#         struct llama_adapter_lora ** adapters,
#         size_t n_adapters,
#         float * scales);
@ctypes_function(
    "llama_set_adapters_lora",
    [llama_context_p_ctypes,
     ctypes.POINTER(llama_adapter_lora_p_ctypes),
     ctypes.c_size_t,
     ctypes.POINTER(ctypes.c_float)
    ],
    ctypes.c_int32,
)
def llama_set_adapters_lora(
    ctx: llama_context_p,
    adapters: Optional[CtypesArray[llama_adapter_lora_p]],
    n_adapters: ctypes.c_size_t,
    scales: Optional[CtypesArray[ctypes.c_float]], /
) -> int:
    """
    Set LoRa adapters on the context.
    Will only modify if the adapters currently in context are different.
    """
    ...


# // Apply a loaded control vector to a llama_context, or if data is NULL, clear
# // the currently loaded vector.
# // n_embd should be the size of a single layer's control, and data should point
# // to an n_embd x n_layers buffer starting from layer 1.
# // il_start and il_end are the layer range the vector should apply to (both inclusive)
# // See llama_control_vector_load in common to load a control vector.
# LLAMA_API int32_t llama_set_adapter_cvec(
#         struct llama_context * ctx,
#                     const float * data,
#                         size_t   len,
#                         int32_t   n_embd,
#                         int32_t   il_start,
#                         int32_t   il_end);
@ctypes_function(
    "llama_set_adapter_cvec",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_float),
        ctypes.c_size_t,
        ctypes.c_int32,
        ctypes.c_int32,
        ctypes.c_int32,
    ],
    ctypes.c_int32,
)
def llama_set_adapter_cvec(
    ctx: llama_context_p,
    data: CtypesPointerOrRef[ctypes.c_float],
    len: int,
    n_embd: int,
    il_start: int,
    il_end: int,
    /,
) -> int:
    """
    Apply a loaded control vector to a llama_context, or if data is NULL, clear
    the currently loaded vector.
    n_embd should be the size of a single layer's control, and data should point
    to an n_embd x n_layers buffer starting from layer 1.
    il_start and il_end are the layer range the vector should apply to (both inclusive)
    See llama_control_vector_load in common to load a control vector.
    """
    ...


# //
# // Memory
# //

# // Clear the memory contents
# // If data == true, the data buffers will also be cleared together with the metadata
# LLAMA_API void llama_memory_clear(
#         llama_memory_t mem,
#                     bool data);
@ctypes_function(
    "llama_memory_clear", [
        llama_memory_i_p_ctypes,
        ctypes.c_bool
    ],
    None,
)
def llama_memory_clear(
    mem: llama_memory_i_p,
    data: bool
):
    ...


# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
# // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
# // seq_id < 0 : match any sequence
# // p0 < 0     : [0,  p1]
# // p1 < 0     : [p0, inf)
# LLAMA_API bool llama_memory_seq_rm(
#         llama_memory_t mem,
#             llama_seq_id seq_id,
#                 llama_pos p0,
#                 llama_pos p1);
@ctypes_function(
    "llama_memory_seq_rm",
    [
        llama_memory_i_p_ctypes,
        llama_seq_id,
        llama_pos,
        llama_pos,
    ],
    ctypes.c_bool,
)
def llama_memory_seq_rm(
    mem: llama_memory_i_p,
    seq_id: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    /,
) -> bool:
    """Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
     Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
     seq_id < 0 : match any sequence
     p0 < 0     : [0,  p1]
     p1 < 0     : [p0, inf)"""
    ...


# // Copy all tokens that belong to the specified sequence to another sequence
# // p0 < 0 : [0,  p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_cp(
#         llama_memory_t mem,
#             llama_seq_id seq_id_src,
#             llama_seq_id seq_id_dst,
#                 llama_pos p0,
#                 llama_pos p1);
@ctypes_function(
    "llama_memory_seq_cp",
    [
        llama_memory_i_p_ctypes,
        llama_seq_id,
        llama_seq_id,
        llama_pos,
        llama_pos,
    ],
    None,
)
def llama_memory_seq_cp(
    mem: llama_memory_i_p,
    seq_id_src: Union[llama_seq_id, int],
    seq_id_dst: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    /,
):
    """Copy all tokens that belong to the specified sequence to another sequence
       p0 < 0 : [0,  p1]
       p1 < 0 : [p0, inf)"""
    ...


# // Removes all tokens that do not belong to the specified sequence
# LLAMA_API void llama_memory_seq_keep(
#         llama_memory_t mem,
#             llama_seq_id seq_id);
@ctypes_function(
    "llama_memory_seq_keep", [llama_memory_i_p_ctypes, llama_seq_id], None
)
def llama_memory_seq_keep(mem: llama_memory_i_p, seq_id: Union[llama_seq_id, int], /):
    """Removes all tokens that do not belong to the specified sequence"""
    ...


# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
# // p0 < 0 : [0,  p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_add(
#         llama_memory_t mem,
#             llama_seq_id seq_id,
#                 llama_pos p0,
#                 llama_pos p1,
#                 llama_pos delta);
@ctypes_function(
    "llama_memory_seq_add",
    [
        llama_memory_i_p_ctypes,
        llama_seq_id,
        llama_pos,
        llama_pos,
        llama_pos,
    ],
    None,
)
def llama_memory_seq_add(
    mem: llama_memory_i_p,
    seq_id: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    delta: Union[llama_pos, int],
    /,
):
    """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
    p0 < 0 : [0,  p1]
    p1 < 0 : [p0, inf)"""
    ...


# // Integer division of the positions by factor of `d > 1`
# // p0 < 0 : [0,  p1]
# // p1 < 0 : [p0, inf)
# LLAMA_API void llama_memory_seq_div(
#         llama_memory_t mem,
#             llama_seq_id seq_id,
#                 llama_pos p0,
#                 llama_pos p1,
#                     int d);
@ctypes_function(
    "llama_memory_seq_div",
    [
        llama_memory_i_p_ctypes,
        llama_seq_id,
        llama_pos,
        llama_pos,
        ctypes.c_int,
    ],
    None,
)
def llama_memory_seq_div(
    mem: llama_memory_i_p,
    seq_id: Union[llama_seq_id, int],
    p0: Union[llama_pos, int],
    p1: Union[llama_pos, int],
    d: Union[ctypes.c_int, int],
    /,
):
    """Integer division of the positions by factor of `d > 1`
       p0 < 0 : [0,  p1]
       p1 < 0 : [p0, inf)"""
    ...


# // Returns the smallest position present in the memory for the specified sequence
# // This is typically non-zero only for SWA caches
# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
# // Return -1 if the sequence is empty
# LLAMA_API llama_pos llama_memory_seq_pos_min(
#         llama_memory_t mem,
#             llama_seq_id seq_id);
@ctypes_function(
    "llama_memory_seq_pos_min",
    [
        llama_memory_i_p_ctypes,
        llama_seq_id,
    ],
    ctypes.c_int32,
)
def llama_memory_seq_pos_min(
    mem: llama_memory_i_p,
    seq_id: Union[llama_seq_id, int]
    ,/) -> int:
    """Returns the smallest position present in the memory for the specified sequence
       This is typically non-zero only for SWA caches
       Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
       Return -1 if the sequence is empty
    """
    ...


# // Returns the largest position present in the memory for the specified sequence
# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
# // Return -1 if the sequence is empty
# LLAMA_API llama_pos llama_memory_seq_pos_max(
#         llama_memory_t mem,
#             llama_seq_id seq_id);
@ctypes_function(
    "llama_memory_seq_pos_max",
    [
        llama_memory_i_p_ctypes,
        llama_seq_id,
    ],
    ctypes.c_int32,
)
def llama_memory_seq_pos_max(
    mem: llama_memory_i_p,
    seq_id: Union[llama_seq_id, int]
    ,/) -> int:
    """Returns the largest position present in the memory for the specified sequence
       This is typically non-zero only for SWA caches
       Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
       Return -1 if the sequence is empty
    """
    ...


# // Check if the memory supports shifting
# LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);
@ctypes_function(
    "llama_memory_can_shift", [llama_memory_i_p_ctypes], ctypes.c_bool)
def llama_memory_can_shift(mem: llama_memory_i_p, /) -> bool:
    ...


# //
# // State / sessions
# //


# // Returns the *actual* size in bytes of the state
# // (logits, embedding and memory)
# // Only use when saving the state, not when restoring it, otherwise the size may be too small.
# LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);
@ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_state_get_size(ctx: llama_context_p, /) -> int:
    """Returns the *actual* size in bytes of the state (rng, logits, embedding and memory) - will often be smaller after compacting tokens"""
    ...


# LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),
#     "use llama_state_get_size instead");
@ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t)
def llama_get_state_size(ctx: llama_context_p, /) -> int:
    """Returns the maximum size in bytes of the state (rng, logits, embedding
    and kv_cache) - will often be smaller after compacting tokens"""
    ...


# // Copies the state to the specified destination address.
# // Destination needs to have allocated enough memory.
# // Returns the number of bytes copied
# LLAMA_API size_t llama_state_get_data(
#         struct llama_context * ctx,
#                      uint8_t * dst,
#                       size_t   size);
@ctypes_function(
    "llama_state_get_data",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
        ctypes.c_size_t,
    ],
    ctypes.c_size_t,
)
def llama_state_get_data(
    ctx: llama_context_p,
    dst: CtypesArray[ctypes.c_uint8],
    size: Union[ctypes.c_size_t, int],
    /,
) -> int:
    """Copies the state to the specified destination address.
    Destination needs to have allocated enough memory.
    Returns the number of bytes copied"""
    ...


# LLAMA_API DEPRECATED(size_t llama_copy_state_data(
#         struct llama_context * ctx,
#                      uint8_t * dst),
#     "use llama_state_get_data instead");
@ctypes_function(
    "llama_copy_state_data",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
    ],
    ctypes.c_size_t,
)
def llama_copy_state_data(
    ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], /
) -> int:
    """Copies the state to the specified destination address.
    Destination needs to have allocated enough memory.
    Returns the number of bytes copied"""
    ...


# // Set the state reading from the specified address
# // Returns the number of bytes read
# LLAMA_API size_t llama_state_set_data(
#         struct llama_context * ctx,
#                const uint8_t * src,
#                       size_t   size);
@ctypes_function(
    "llama_state_set_data",
    [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t],
    ctypes.c_size_t,
)
def llama_state_set_data(
    ctx: llama_context_p,
    src: CtypesArray[ctypes.c_uint8],
    size: Union[ctypes.c_size_t, int],
    /,
) -> int:
    """Set the state reading from the specified address
    Returns the number of bytes read"""
    ...


# LLAMA_API DEPRECATED(size_t llama_set_state_data(
#         struct llama_context * ctx,
#                const uint8_t * src),
#     "use llama_state_set_data instead");
@ctypes_function(
    "llama_set_state_data",
    [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)],
    ctypes.c_size_t,
)
def llama_set_state_data(
    ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], /
) -> int:
    """Set the state reading from the specified address"""
    ...


# Save/load session file
# LLAMA_API bool llama_state_load_file(
#         struct llama_context * ctx,
#                   const char * path_session,
#                  llama_token * tokens_out,
#                       size_t   n_token_capacity,
#                       size_t * n_token_count_out);
@ctypes_function(
    "llama_state_load_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.POINTER(ctypes.c_size_t),
    ],
    ctypes.c_bool,
)
def llama_state_load_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens_out: CtypesArray[llama_token],
    n_token_capacity: Union[ctypes.c_size_t, int],
    n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
    /,
) -> bool:
    ...


# LLAMA_API DEPRECATED(bool llama_load_session_file(
#         struct llama_context * ctx,
#                   const char * path_session,
#                  llama_token * tokens_out,
#                       size_t   n_token_capacity,
#                       size_t * n_token_count_out),
#     "use llama_state_load_file instead");
@ctypes_function(
    "llama_load_session_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.POINTER(ctypes.c_size_t),
    ],
    ctypes.c_bool,
)
def llama_load_session_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens_out: CtypesArray[llama_token],
    n_token_capacity: Union[ctypes.c_size_t, int],
    n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
    /,
) -> bool:
    ...


# LLAMA_API bool llama_state_save_file(
#         struct llama_context * ctx,
#                   const char * path_session,
#            const llama_token * tokens,
#                       size_t   n_token_count);
@ctypes_function(
    "llama_state_save_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
    ],
    ctypes.c_bool,
)
def llama_state_save_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens: CtypesArray[llama_token],
    n_token_count: Union[ctypes.c_size_t, int],
    /,
) -> bool:
    ...


# LLAMA_API DEPRECATED(bool llama_save_session_file(
#         struct llama_context * ctx,
#                   const char * path_session,
#            const llama_token * tokens,
#                       size_t   n_token_count),
#     "use llama_state_save_file instead");
@ctypes_function(
    "llama_save_session_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_token_p,
        ctypes.c_size_t,
    ],
    ctypes.c_bool,
)
def llama_save_session_file(
    ctx: llama_context_p,
    path_session: bytes,
    tokens: CtypesArray[llama_token],
    n_token_count: Union[ctypes.c_size_t, int],
    /,
) -> bool:
    ...


# // Get the exact size needed to copy the state of a single sequence
# LLAMA_API size_t llama_state_seq_get_size(
#         struct llama_context * ctx,
#                 llama_seq_id   seq_id);
@ctypes_function(
    "llama_state_seq_get_size",
    [llama_context_p_ctypes, llama_seq_id],
    ctypes.c_size_t,
)
def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> int:
    """Get the exact size needed to copy the state of a single sequence"""
    ...


# // Copy the state of a single sequence into the specified buffer
# LLAMA_API size_t llama_state_seq_get_data(
#         struct llama_context * ctx,
#                      uint8_t * dst,
#                       size_t   size,
#                 llama_seq_id   seq_id);
@ctypes_function(
    "llama_state_seq_get_data",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
        ctypes.c_size_t,
        llama_seq_id,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_get_data(
    ctx: llama_context_p,
    dst: CtypesArray[ctypes.c_uint8],
    size: Union[ctypes.c_size_t, int],
    seq_id: llama_seq_id,
    /,
) -> int:
    """Copy the state of a single sequence into the specified buffer"""
    ...


# // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence
# // Returns:
# //  - Positive: Ok
# //  - Zero: Failed to load
# LLAMA_API size_t llama_state_seq_set_data(
#         struct llama_context * ctx,
#                const uint8_t * src,
#                       size_t   size,
#                 llama_seq_id   dest_seq_id);
@ctypes_function(
    "llama_state_seq_set_data",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
        ctypes.c_size_t,
        llama_seq_id,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_set_data(
    ctx: llama_context_p,
    src: CtypesArray[ctypes.c_uint8],
    size: Union[ctypes.c_size_t, int],
    dest_seq_id: llama_seq_id,
    /,
) -> int:
    """Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence"""
    ...


# LLAMA_API size_t llama_state_seq_save_file(
#         struct llama_context * ctx,
#                   const char * filepath,
#                 llama_seq_id   seq_id,
#            const llama_token * tokens,
#                       size_t   n_token_count);
@ctypes_function(
    "llama_state_seq_save_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_seq_id,
        llama_token_p,
        ctypes.c_size_t,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_save_file(
    ctx: llama_context_p,
    filepath: bytes,
    seq_id: llama_seq_id,
    tokens: CtypesArray[llama_token],
    n_token_count: Union[ctypes.c_size_t, int],
    /,
) -> int:
    ...

# If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded
# LLAMA_API size_t llama_state_seq_load_file(
#         struct llama_context * ctx,
#                   const char * filepath,
#                 llama_seq_id   dest_seq_id,
#                  llama_token * tokens_out,
#                       size_t   n_token_capacity,
#                       size_t * n_token_count_out);
@ctypes_function(
    "llama_state_seq_load_file",
    [
        llama_context_p_ctypes,
        ctypes.c_char_p,
        llama_seq_id,
        llama_token_p,
        ctypes.c_size_t,
        ctypes.POINTER(ctypes.c_size_t),
    ],
    ctypes.c_size_t,
)
def llama_state_seq_load_file(
    ctx: llama_context_p,
    filepath: bytes,
    dest_seq_id: llama_seq_id,
    tokens_out: CtypesArray[llama_token],
    n_token_capacity: Union[ctypes.c_size_t, int],
    n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t],
    /,
) -> int:
    """
    If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded
    """
    ...

# define LLAMA_STATE_SEQ_FLAGS_NONE 0
LLAMA_STATE_SEQ_FLAGS_NONE = 0

# // for backwards-compat
LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1

# // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)
LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1

# // keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load)
# // Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag.
LLAMA_STATE_SEQ_FLAGS_ON_DEVICE = 2

llama_state_seq_flags = ctypes.c_uint32

# LLAMA_API size_t llama_state_seq_get_size_ext(
#         struct llama_context * ctx,
#                 llama_seq_id   seq_id,
#         llama_state_seq_flags   flags);
@ctypes_function(
    "llama_state_seq_get_size_ext",
    [
        llama_context_p_ctypes,
        llama_seq_id,
        llama_state_seq_flags,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_get_size_ext(
    ctx: llama_context_p,
    seq_id: llama_seq_id,
    flags: llama_state_seq_flags,
    /,
) -> int:
    ...


# LLAMA_API size_t llama_state_seq_get_data_ext(
#         struct llama_context * ctx,
#                         uint8_t * dst,
#                         size_t   size,
#                 llama_seq_id   seq_id,
#         llama_state_seq_flags   flags);
@ctypes_function(
    "llama_state_seq_get_data_ext",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
        ctypes.c_size_t,
        llama_seq_id,
        llama_state_seq_flags,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_get_data_ext(
    ctx: llama_context_p,
    dst: CtypesPointer[ctypes.c_uint8],
    size: Union[int, ctypes.c_size_t],
    seq_id: llama_seq_id,
    flags: llama_state_seq_flags,
    /,
) -> int:
    ...


# LLAMA_API size_t llama_state_seq_set_data_ext(
#         struct llama_context * ctx,
#                 const uint8_t * src,
#                         size_t   size,
#                 llama_seq_id   dest_seq_id,
#         llama_state_seq_flags   flags);
@ctypes_function(
    "llama_state_seq_set_data_ext",
    [
        llama_context_p_ctypes,
        ctypes.POINTER(ctypes.c_uint8),
        ctypes.c_size_t,
        llama_seq_id,
        llama_state_seq_flags,
    ],
    ctypes.c_size_t,
)
def llama_state_seq_set_data_ext(
    ctx: llama_context_p,
    src: CtypesPointer[ctypes.c_uint8],
    size: Union[int, ctypes.c_size_t],
    dest_seq_id: llama_seq_id,
    flags: llama_state_seq_flags,
    /,
) -> int:
    ...


# //
# // Decoding
# //


# // Return batch for single sequence of tokens
# // The sequence ID will be fixed to 0
# // The position of the tokens will be tracked automatically by llama_decode
# //
# // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
# //
# LLAMA_API struct llama_batch llama_batch_get_one(
#               llama_token * tokens,
#                   int32_t   n_tokens);
@ctypes_function(
    "llama_batch_get_one",
    [
        llama_token_p,
        ctypes.c_int32,
    ],
    llama_batch,
)
def llama_batch_get_one(
    tokens: CtypesArray[llama_token],
    n_tokens: Union[ctypes.c_int, int],
    /,
) -> llama_batch:
    """Return batch for single sequence of tokens starting at pos_0

    NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
    """
    ...


# // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
# // Each token can be assigned up to n_seq_max sequence ids
# // The batch has to be freed with llama_batch_free()
# // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
# // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
# // The rest of the llama_batch members are allocated with size n_tokens
# // All members are left uninitialized
# LLAMA_API struct llama_batch llama_batch_init(
#         int32_t n_tokens,
#         int32_t embd,
#         int32_t n_seq_max);
@ctypes_function(
    "llama_batch_init", [ctypes.c_int32, ctypes.c_int32, ctypes.c_int32], llama_batch
)
def llama_batch_init(
    n_tokens: ctypes.c_int32,
    embd: ctypes.c_int32,
    n_seq_max: ctypes.c_int32,
    /,
) -> llama_batch:
    """Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
    Each token can be assigned up to n_seq_max sequence ids
    The batch has to be freed with llama_batch_free()
    If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
    Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
    The rest of the llama_batch members are allocated with size n_tokens
    All members are left uninitialized"""
    ...


# // Frees a batch of tokens allocated with llama_batch_init()
# LLAMA_API void llama_batch_free(struct llama_batch batch);
@ctypes_function("llama_batch_free", [llama_batch], None)
def llama_batch_free(batch: llama_batch, /):
    """Frees a batch of tokens allocated with llama_batch_init()"""
    ...


# // Process a batch of tokens.
# // In contrast to llama_decode() - this call does not use KV cache.
# // For encode-decoder contexts, processes the batch using the encoder.
# // Can store the encoder output internally for later use by the decoder's cross-attention layers.
# //   0 - success
# // < 0 - error. the memory state is restored to the state before this call
# LLAMA_API int32_t llama_encode(
#         struct llama_context * ctx,
#           struct llama_batch   batch);
@ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int:
    """Processes a batch of tokens with the ecoder part of the encoder-decoder model.
    Stores the encoder output internally for later use by the decoder cross-attention layers.
    0 - success
    < 0 - error"""
    ...


# // Process a batch of tokens.
# // Requires the context to have a memory.
# // For encode-decoder contexts, processes the batch using the decoder.
# // Positive return values does not mean a fatal error, but rather a warning.
# // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context
# //   To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()
# // Upon other return values, the memory state is restored to the state before this call
# //    0 - success
# //    1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
# //    2 - aborted     (processed ubatches will remain in the context's memory)
# //   -1 - invalid input batch
# // < -1 - fatal error (processed ubatches will remain in the context's memory)
# LLAMA_API int32_t llama_decode(
#         struct llama_context * ctx,
#             struct llama_batch   batch);
@ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32)
def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int:
    """Positive return values does not mean a fatal error, but rather a warning.
       0 - success
       1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
       2 - aborted     (processed ubatches will remain in the context's memory)
      -1 - invalid input batch
    < -1 - fatal error (processed ubatches will remain in the context's memory)
    """
    ...


# // Extended batch API

# struct llama_embd {
#     const float * data;
#     size_t n_rows; // number of embedding rows in data
#     size_t n_embd; // size of one row
# };
class llama_embd(ctypes.Structure):
    """Embedding view containing n_rows rows of n_embd floats."""

    if TYPE_CHECKING:
        data: ctypes.POINTER(ctypes.c_float) # type: ignore
        n_rows: int
        n_embd: int

    _fields_ = [
        ("data", ctypes.POINTER(ctypes.c_float)),
        ("n_rows", ctypes.c_size_t),
        ("n_embd", ctypes.c_size_t),
    ]

# LLAMA_API struct llama_batch_ext * llama_batch_ext_init (struct llama_context * ctx);
@ctypes_function("llama_batch_ext_init", [llama_context_p_ctypes], llama_batch_ext_p_ctypes)
def llama_batch_ext_init(ctx: llama_context_p, /) -> llama_batch_ext_p:
    """Create a batch for this context; release it with llama_batch_ext_free()."""
    ...


# LLAMA_API void llama_batch_ext_free (struct llama_batch_ext * batch);
@ctypes_function("llama_batch_ext_free", [llama_batch_ext_p_ctypes], None)
def llama_batch_ext_free(batch: llama_batch_ext_p, /):
    ...


# LLAMA_API void llama_batch_ext_clear(struct llama_batch_ext * batch);
@ctypes_function("llama_batch_ext_clear", [llama_batch_ext_p_ctypes], None)
def llama_batch_ext_clear(batch: llama_batch_ext_p, /):
    """Remove all entries so the batch can be reused."""
    ...


# // Add an input token to the batch, with default values:
# //     id = LLAMA_TOKEN_NULL
# //     embd = nullptr
# //     pos = not set, the caller must set it with llama_batch_ext_set_pos()
# // Returns the batch index (>= 0)
# // On error:
# //     -1: batch is full
# //     -2: token is invalid (id == LLAMA_TOKEN_NULL or invalid embd)
# //     -3: invalid sequence id
# LLAMA_API int32_t llama_batch_ext_add      (struct llama_batch_ext * batch, llama_seq_id seq_id);
@ctypes_function("llama_batch_ext_add", [llama_batch_ext_p_ctypes, llama_seq_id], ctypes.c_int32)
def llama_batch_ext_add(batch: llama_batch_ext_p, seq_id: int, /) -> int:
    """Add an entry without a token or embedding. Returns its index or an error."""
    ...


# // Add an input token to the batch, with a specified token ID or token embedding
# LLAMA_API int32_t llama_batch_ext_add_token(struct llama_batch_ext * batch, llama_seq_id seq_id, llama_token id);
@ctypes_function(
    "llama_batch_ext_add_token",
    [llama_batch_ext_p_ctypes, llama_seq_id, llama_token],
    ctypes.c_int32,
)
def llama_batch_ext_add_token(batch: llama_batch_ext_p, seq_id: int, id: int, /) -> int:
    """Add a token. Returns its index, or -1 full, -2 invalid input, -3 invalid sequence.

    On -2, an empty entry remains in the batch; clear it before reuse.
    """
    ...


# LLAMA_API int32_t llama_batch_ext_add_embd (struct llama_batch_ext * batch, llama_seq_id seq_id, struct llama_embd embd);
@ctypes_function(
    "llama_batch_ext_add_embd",
    [llama_batch_ext_p_ctypes, llama_seq_id, llama_embd],
    ctypes.c_int32,
)
def llama_batch_ext_add_embd(batch: llama_batch_ext_p, seq_id: int, embd: llama_embd, /) -> int:
    """Copy an embedding into the batch. Returns its index or a negative error.

    On -2, an empty entry remains in the batch; clear it before reuse.
    """
    ...


# // Add the token at index idx in the batch to another sequence id. The position will stays the same.
# // Note: this should be called before other _set() functions
# LLAMA_API bool llama_batch_ext_add_seq(
#                             struct llama_batch_ext * batch,
#                                             int32_t   idx,
#                                         llama_seq_id   seq_id);
@ctypes_function(
    "llama_batch_ext_add_seq",
    [llama_batch_ext_p_ctypes, ctypes.c_int32, llama_seq_id],
    ctypes.c_bool,
)
def llama_batch_ext_add_seq(batch: llama_batch_ext_p, idx: int, seq_id: int, /) -> bool:
    """Add an entry to another sequence before calling the other setters."""
    ...


# // Set the token embedding for the token at index idx in the batch
# // use it after llama_batch_ext_add_token() to have an entry with both a token id and an embedding
# LLAMA_API bool llama_batch_ext_set_embd_token(
#                             struct llama_batch_ext * batch,
#                                             int32_t   idx,
#                                     struct llama_embd   embd);
@ctypes_function(
    "llama_batch_ext_set_embd_token",
    [llama_batch_ext_p_ctypes, ctypes.c_int32, llama_embd],
    ctypes.c_bool,
)
def llama_batch_ext_set_embd_token(batch: llama_batch_ext_p, idx: int, embd: llama_embd, /) -> bool:
    """Copy a token embedding into an existing entry."""
    ...


# // Set the "state" embedding for the token at index idx in the batch
# // "state" here means extra hidden state carried over from a previous stage, e.g.:
# //   - MTP: state from N layers of the target model
# //   - Qwen3 VL (deepstack): state from N layers of the vision encoder
# //   - LiliCorr: context embeddings for the transformers
# // Returns false if the context does not take a state embedding (MTP and LiliCorr contexts do)
# LLAMA_API bool llama_batch_ext_set_embd_state(
#                             struct llama_batch_ext * batch,
#                                             int32_t   idx,
#                                     struct llama_embd   embd);
@ctypes_function(
    "llama_batch_ext_set_embd_state",
    [llama_batch_ext_p_ctypes, ctypes.c_int32, llama_embd],
    ctypes.c_bool,
)
def llama_batch_ext_set_embd_state(batch: llama_batch_ext_p, idx: int, embd: llama_embd, /) -> bool:
    """Set hidden state; return False for unsupported contexts or invalid input.

    Currently MTP contexts accept states with the model's output embedding width.
    """
    ...


# // Set if output embeddings should be available for the token at index idx in the batch
# // Note: for now, this is equivalent to setting the output logits
# LLAMA_API bool llama_batch_ext_set_output_embd(
#                             struct llama_batch_ext * batch,
#                                             int32_t  idx,
#                                                 bool  value);
@ctypes_function(
    "llama_batch_ext_set_output_embd",
    [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.c_bool],
    ctypes.c_bool,
)
def llama_batch_ext_set_output_embd(batch: llama_batch_ext_p, idx: int, value: bool, /) -> bool:
    """Select embedding output for an entry; currently also selects logits."""
    ...


# // Set output logits for the token at index idx in the batch
# // Note: for now, this is equivalent to setting the output embd
# LLAMA_API bool llama_batch_ext_set_output_logits(
#                             struct llama_batch_ext * batch,
#                                             int32_t  idx,
#                                                 bool  value);
@ctypes_function(
    "llama_batch_ext_set_output_logits",
    [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.c_bool],
    ctypes.c_bool,
)
def llama_batch_ext_set_output_logits(batch: llama_batch_ext_p, idx: int, value: bool, /) -> bool:
    """Select logits for an entry; currently also selects embedding output."""
    ...


# // Set custom position for the token at index idx in the batch
# // For M-RoPE models:
# //     - Embedding tokens must have multiple positions per token
# //     - Text token only requires one single position per token
# LLAMA_API bool llama_batch_ext_set_pos(
#                             struct llama_batch_ext * batch,
#                                             int32_t   idx,
#                                     const llama_pos * pos);
@ctypes_function(
    "llama_batch_ext_set_pos",
    [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.POINTER(llama_pos)],
    ctypes.c_bool,
)
def llama_batch_ext_set_pos(
    batch: llama_batch_ext_p, idx: int, pos: CtypesPointerOrRef[llama_pos], /
) -> bool:
    """Set one position for text, or all M-RoPE positions for an embedding."""
    ...


# // Return values are the same as llama_decode()
# LLAMA_API int32_t llama_process(
#                             struct llama_context * ctx,
#                             enum llama_process_type   type,
#                             struct llama_batch_ext * batch);
@ctypes_function(
    "llama_process",
    [llama_context_p_ctypes, ctypes.c_int, llama_batch_ext_p_ctypes],
    ctypes.c_int32,
)
def llama_process(ctx: llama_context_p, type: int, batch: llama_batch_ext_p, /) -> int:
    """Encode or decode an extended batch. Return codes match llama_decode()."""
    ...


# // Set the number of threads used for decoding
# // n_threads is the number of threads used for generation (single token)
# // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);
@ctypes_function(
    "llama_set_n_threads",
    [
        llama_context_p_ctypes,
        ctypes.c_int32,
        ctypes.c_int32,
    ],
    None,
)
def llama_set_n_threads(
    ctx: llama_context_p,
    n_threads: Union[ctypes.c_int32, int],
    n_threads_batch: Union[ctypes.c_int32, int],
    /,
):
    """Set the number of threads used for decoding
    n_threads is the number of threads used for generation (single token)
    n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
    """
    ...


# // Get the number of threads used for generation of a single token.
# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);
@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32)
def llama_n_threads(ctx: llama_context_p, /) -> int:
    """Get the number of threads used for generation of a single token"""
    ...


# // Get the number of threads used for prompt and batch processing (multiple token).
# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);
@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32)
def llama_n_threads_batch(ctx: llama_context_p, /) -> int:
    """Get the number of threads used for prompt and batch processing (multiple token)"""
    ...


# // Set whether the model is in embeddings mode or not
# // TODO: rename to avoid confusion with llama_get_embeddings()
# LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);
@ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /):
    """
    Set whether the model is in embeddings model or not
    """
    ...


# // Set whether to use causal attention or not
# // If set to true, the model will only attend to the past tokens
# LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);
@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /):
    """Set whether to use causal attention or not
    If set to true, the model will only attend to the past tokens"""
    ...


# // Returns whether the context is currently using causal attention
# LLAMA_API bool llama_get_causal_attn(const struct llama_context * ctx);
@ctypes_function("llama_get_causal_attn", [llama_context_p_ctypes], ctypes.c_bool)
def llama_get_causal_attn(ctx: llama_context_p, /) -> bool:
    """Returns whether the context is currently using causal attention"""
    ...


# // Set whether the model is in warmup mode or not
# // If true, all model tensors are activated during llama_decode() to load and cache their weights.
# //
# // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models,
# //       so it is generally not recommended to use in practice. will be removed in the future
# DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup),
#         "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]");
@ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None)
def llama_set_warmup(ctx: llama_context_p, warmup: bool, /):
    """DEPRECATED: using this can cause extra graph reallocations because it changes the graph topology with MoE models,
    so it is generally not recommended to use in practice. will be removed in the future"""
    ...

# // Set abort callback
# LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);
@ctypes_function(
    "llama_set_abort_callback",
    [llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p],
    None,
)
def llama_set_abort_callback(
    ctx: llama_context_p,
    abort_callback: ggml_abort_callback,
    abort_callback_data: Optional[ctypes.c_void_p],
    /,
):
    """Set or clear the context's borrowed native abort callback."""
    ...


# // Wait until all computations are finished
# // This is automatically done when using one of the functions below to obtain the computation results
# // and is not necessary to call it explicitly in most cases
# LLAMA_API void llama_synchronize(struct llama_context * ctx);
@ctypes_function("llama_synchronize", [llama_context_p_ctypes], None)
def llama_synchronize(ctx: llama_context_p, /):
    """Wait until all computations are finished
    This is automatically done when using one of the functions below to obtain the computation results
    and is not necessary to call it explicitly in most cases"""
    ...


# // Token logits obtained from the last call to llama_decode()
# // The logits for which llama_batch.logits[i] != 0 are stored contiguously
# // in the order they have appeared in the batch.
# // Rows: number of tokens for which llama_batch.logits[i] != 0
# // Cols: n_vocab
# // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
# LLAMA_API float * llama_get_logits(struct llama_context * ctx);
@ctypes_function(
    "llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
    """Token logits obtained from the last call to llama_decode()
    The logits for which llama_batch.logits[i] != 0 are stored contiguously
    in the order they have appeared in the batch.
    Rows: number of tokens for which llama_batch.logits[i] != 0
    Cols: n_vocab

    Returns:
        Pointer to the logits buffer of shape (n_tokens, n_vocab)"""
    ...


# // Logits for the ith token. For positive indices, Equivalent to:
# // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab
# // Negative indices can be used to access logits in reverse order, -1 is the last logit.
# // returns NULL for invalid ids.
# LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_logits_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_logits_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesPointer[ctypes.c_float]:
    """Logits for the ith token. Equivalent to:
    llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab"""
    ...


# // Get all output token embeddings.
# // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
# // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
# // in the order they have appeared in the batch.
# // shape: [n_outputs*n_embd]
# // Otherwise, returns NULL.
# // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
# LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
@ctypes_function(
    "llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float)
)
def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]:
    """Get the embeddings for the input
    shape: [n_embd] (1-dimensional)"""
    ...


# // Get the embeddings for the ith token. For positive indices, Equivalent to:
# // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd
# // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding.
# // shape: [n_embd] (1-dimensional)
# // returns NULL for invalid ids.
# LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_embeddings_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_ith(
    ctx: llama_context_p, i: Union[ctypes.c_int32, int], /
) -> CtypesArray[ctypes.c_float]:
    """Get the embeddings for the ith sequence
    llama_get_embeddings(ctx) + i*n_embd"""
    ...


# // Get the embeddings for a sequence id
# // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[1] with the rank of the sequence
# // otherwise: float[n_embd] (1-dimensional)
# LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);
@ctypes_function(
    "llama_get_embeddings_seq",
    [llama_context_p_ctypes, llama_seq_id],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_embeddings_seq(
    ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /
) -> CtypesArray[ctypes.c_float]:
    """Get the embeddings for a sequence id
    Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
    shape: [n_embd] (1-dimensional)"""
    ...


# //
# // backend sampling API [EXPERIMENTAL]
# // note: use only if the llama_context was created with at least one llama_sampler_seq_config
# //

# // Get the backend sampled token for the ith token.
# // With multiple outputs, sampler state advances when the token is accepted,
# // not when it is read through this function.
# // When accepting multiple outputs, accept a contiguous prefix in output order.
# // Returns LLAMA_TOKEN_NULL if no token was sampled.
# LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_token_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    llama_token,
)
def llama_get_sampled_token_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_int32:
    """
    Get the backend sampled token for the ith token.
    With multiple outputs, sampler state advances when the token is accepted,
    not when it is read through this function.
    When accepting multiple outputs, accept a contiguous prefix in output order.
    Returns LLAMA_TOKEN_NULL if no token was sampled.
    """
    ...


# // Get the backend sampled probabilities for the ith token
# // The index matches llama_get_sampled_token_ith().
# // Returns NULL if no probabilities were generated.
# LLAMA_API float *  llama_get_sampled_probs_ith      (struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_probs_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_sampled_probs_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesArray[ctypes.c_float]:
    """
    Get the backend sampled probabilities for the ith token
    The index matches llama_get_sampled_token_ith().
    Returns NULL if no probabilities were generated.
    """
    ...


# LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_probs_count_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.c_uint32,
)
def llama_get_sampled_probs_count_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_uint32:
    """
    Get the backend sampled probabilites count for the ith token
    """
    ...


# // Get the backend sampled logits for the ith token
# // Returns NULL if no logits were sampled.
# LLAMA_API float *  llama_get_sampled_logits_ith      (struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_logits_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
)
def llama_get_sampled_logits_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesArray[ctypes.c_float]:
    """
    Get the backend sampled logits for the ith token
    Returns NULL if no logits were sampled.
    """
    ...


# LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_logits_count_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.c_uint32,
)
def llama_get_sampled_logits_count_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_uint32:
    """
    Get the backend sampled logits count for the ith token
    """
    ...


# // Get the backend sampled candidates (token ids) for the ith token
# // These are needed to map probability/logit indices to vocab token ids.
# // Returns NULL if no candidates were sampled.
# LLAMA_API llama_token * llama_get_sampled_candidates_ith      (struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_candidates_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(llama_token),
)
def llama_get_sampled_candidates_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> CtypesArray[llama_token]:
    """
    Get the backend sampled candidates (token ids) for the ith token
    These are needed to map probability/logit indices to vocab token ids.
    Returns NULL if no candidates were sampled.
    """
    ...


# LLAMA_API uint32_t      llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);
@ctypes_function(
    "llama_get_sampled_candidates_count_ith",
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.c_uint32,
)
def llama_get_sampled_candidates_count_ith(
    ctx: llama_context_p, i: ctypes.c_int32, /
) -> ctypes.c_uint32:
    """
    Get the backend sampled candidates (token ids) count for the ith token
    """
    ...


# //
# // Vocab
# //


# LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
    "llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p
)
def llama_vocab_get_text(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bytes:
    ...


# LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
    "llama_vocab_get_score", [llama_vocab_p_ctypes, llama_token], ctypes.c_float
)
def llama_vocab_get_score(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> float:
    ...


# LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
    "llama_vocab_get_attr", [llama_vocab_p_ctypes, llama_token], ctypes.c_int
)
def llama_vocab_get_attr(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> int:
    ...


# // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)
# LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
    "llama_vocab_is_eog", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool
)
def llama_vocab_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool:
    """Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)"""
    ...


# // Identify if Token Id is a control token or a render-able token
# LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);
@ctypes_function(
    "llama_vocab_is_control", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool
)
def llama_vocab_is_control(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bool:
    """Identify if Token Id is a control token or a render-able token"""
    ...


# // Special tokens


# LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence
@ctypes_function("llama_vocab_bos", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_bos(vocab: llama_vocab_p, /) -> llama_token:
    """beginning-of-sentence"""
    ...


# LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence
@ctypes_function("llama_vocab_eos", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_eos(vocab: llama_vocab_p, /) -> llama_token:
    """end-of-sentence"""
    ...


# LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn
@ctypes_function("llama_vocab_eot", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_eot(vocab: llama_vocab_p, /) -> llama_token:
    """end-of-turn"""
    ...


# LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator
@ctypes_function("llama_vocab_sep", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_sep(vocab: llama_vocab_p, /) -> llama_token:
    """sentence separator"""
    ...


# LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line
@ctypes_function("llama_vocab_nl", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_nl(vocab: llama_vocab_p, /) -> llama_token:
    """next-line"""
    ...


# LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding
@ctypes_function("llama_vocab_pad", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token:
    """padding"""
    ...


# LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask
@ctypes_function("llama_vocab_mask", [llama_vocab_p_ctypes], llama_token)
def llama_vocab_mask(vocab: llama_vocab_p, /) -> llama_token:
    """mask"""
    ...


# LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_get_add_bos",
    [llama_vocab_p_ctypes],
    ctypes.c_bool,
)
def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool:
    ...


# LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_get_add_eos",
    [llama_vocab_p_ctypes],
    ctypes.c_bool,
)
def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool:
    ...


# LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_get_add_sep",
    [llama_vocab_p_ctypes],
    ctypes.c_bool,
)
def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool:
    ...


# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)
# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens);
@ctypes_function(
    "llama_vocab_get_suppress_tokens",
    [
        llama_vocab_p_ctypes,
        ctypes.POINTER(ctypes.c_int32),
    ],
    llama_token_p,
)
def llama_vocab_get_suppress_tokens(
    vocab: llama_vocab_p,
    n_suppress_tokens: ctypes.POINTER(ctypes.c_int32),  # type: ignore
) -> llama_token_p:  # type: ignore
    """
    model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)
    """
    ...


# LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_fim_pre",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token:
    ...


# LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_fim_suf",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token:
    ...


# LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_fim_mid",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token:
    ...


# LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_fim_pad",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token:
    ...


# LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_fim_rep",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token:
    ...


# LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_vocab_fim_sep",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token:
    ...



# DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");
@ctypes_function(
    "llama_token_get_text",
    [llama_vocab_p_ctypes, llama_token],
    ctypes.c_char_p,
)
def llama_token_get_text(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bytes:
    ...


# DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");
@ctypes_function(
    "llama_token_get_score",
    [llama_vocab_p_ctypes, llama_token],
    ctypes.c_float,
)
def llama_token_get_score(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> float:
    ...

# DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");
@ctypes_function(
    "llama_token_get_attr",
    [llama_vocab_p_ctypes, llama_token],
    ctypes.c_int,
)
def llama_token_get_attr(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> int:
    ...

# DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");
@ctypes_function(
    "llama_token_is_eog",
    [llama_vocab_p_ctypes, llama_token],
    ctypes.c_bool,
)
def llama_token_is_eog(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bool:
    ...

# DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");
@ctypes_function(
    "llama_token_is_control",
    [llama_vocab_p_ctypes, llama_token],
    ctypes.c_bool,
)
def llama_token_is_control(
    vocab: llama_vocab_p, token: Union[llama_token, int], /
) -> bool:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");
@ctypes_function(
    "llama_token_bos",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_bos(vocab: llama_vocab_p, /) -> int:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");
@ctypes_function(
    "llama_token_eos",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_eos(vocab: llama_vocab_p, /) -> int:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");
@ctypes_function(
    "llama_token_eot",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_eot(vocab: llama_vocab_p, /) -> int:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");
@ctypes_function(
    "llama_token_cls",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_cls(vocab: llama_vocab_p, /) -> int:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");
@ctypes_function(
    "llama_token_sep",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_sep(vocab: llama_vocab_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");
@ctypes_function(
    "llama_token_nl",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_nl(vocab: llama_vocab_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");
@ctypes_function(
    "llama_token_pad",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_pad(vocab: llama_vocab_p, /) -> int:
    ...


# DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");
@ctypes_function(
    "llama_add_bos_token",
    [llama_vocab_p_ctypes],
    ctypes.c_bool,
)
def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool:
    ...

# DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");
@ctypes_function(
    "llama_add_eos_token",
    [llama_vocab_p_ctypes],
    ctypes.c_bool,
)
def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool:
    ...


# DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");
@ctypes_function(
    "llama_token_fim_pre",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");
@ctypes_function(
    "llama_token_fim_suf",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");
@ctypes_function(
    "llama_token_fim_mid",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");
@ctypes_function(
    "llama_token_fim_pad",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");
@ctypes_function(
    "llama_token_fim_rep",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token:
    ...

# DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");
@ctypes_function(
    "llama_token_fim_sep",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token:
    ...

# // CLS is equivalent to BOS
# DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification
#         "use llama_vocab_bos instead");
@ctypes_function(
    "llama_vocab_cls",
    [llama_vocab_p_ctypes],
    llama_token,
)
def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token:
    ...


# //
# // Tokenization
# //
# // The API is thread-safe.
# //


# /// @details Convert the provided text into tokens.
# /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
# /// @return Returns the number of tokens on success, no more than n_tokens_max
# /// @return Returns a negative number on failure - the number of tokens that would have been returned
# /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)
# /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.
# /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated
# ///                      as plaintext. Does not insert a leading space.
# LLAMA_API int32_t llama_tokenize(
#     const struct llama_vocab * vocab,
#                   const char * text,
#                      int32_t   text_len,
#                  llama_token * tokens,
#                      int32_t   n_tokens_max,
#                         bool   add_special,
#                         bool   parse_special);
@ctypes_function(
    "llama_tokenize",
    [
        llama_vocab_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_int32,
        llama_token_p,
        ctypes.c_int32,
        ctypes.c_bool,
        ctypes.c_bool,
    ],
    ctypes.c_int32,
)
def llama_tokenize(
    vocab: llama_vocab_p,
    text: bytes,
    text_len: Union[ctypes.c_int, int],
    tokens: CtypesArray[llama_token],
    n_tokens_max: Union[ctypes.c_int, int],
    add_special: Union[ctypes.c_bool, bool],
    parse_special: Union[ctypes.c_bool, bool],
    /,
) -> int:
    """Convert the provided text into tokens.

    Args:
        vocab: The vocabulary to use for tokenization.
        text: The text to tokenize.
        text_len: The length of the text.
        tokens: The tokens pointer must be large enough to hold the resulting tokens.
        n_max_tokens: The maximum number of tokens to return.
        add_special: Allow adding special tokenns if the model is configured to do so.
        parse_special: Allow parsing special tokens.

    Returns:
        Returns the number of tokens on success, no more than n_tokens_max
        Returns a negative number on failure - the number of tokens that would have been returned
    """
    ...


# // Token Id -> Piece.
# // Uses the vocabulary in the provided context.
# // Does not write null terminator to the buffer.
# // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')
# // @param special If true, special tokens are rendered in the output.
# LLAMA_API int32_t llama_token_to_piece(
#           const struct llama_vocab * vocab,
#                        llama_token   token,
#                               char * buf,
#                            int32_t   length,
#                            int32_t   lstrip,
#                               bool   special);
@ctypes_function(
    "llama_token_to_piece",
    [
        llama_vocab_p_ctypes,
        llama_token,
        ctypes.c_char_p,
        ctypes.c_int32,
        ctypes.c_int32,
        ctypes.c_bool,
    ],
    ctypes.c_int32,
)
def llama_token_to_piece(
    vocab: llama_vocab_p,
    token: Union[llama_token, int],
    buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]],
    length: Union[ctypes.c_int, int],
    lstrip: Union[ctypes.c_int, int],
    special: Union[ctypes.c_bool, bool],
    /,
) -> int:
    """Token Id -> Piece.
    Uses the vocabulary in the provided context.
    Does not write null terminator to the buffer.
    User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.

    Args:
        vocab: The vocabulary to use for tokenization.
        token: The token to convert.
        buf: The buffer to write the token to.
        length: The length of the buffer.
        lstrip: The number of leading spaces to skip.
        special: If true, special tokens are rendered in the output."""
    ...


# /// @details Convert the provided tokens into text (inverse of llama_tokenize()).
# /// @param text The char pointer must be large enough to hold the resulting text.
# /// @return Returns the number of chars/bytes on success, no more than text_len_max.
# /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.
# /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.
# /// @param unparse_special If true, special tokens are rendered in the output.
# LLAMA_API int32_t llama_detokenize(
#     const struct llama_vocab * vocab,
#             const llama_token * tokens,
#                         int32_t   n_tokens,
#                         char * text,
#                         int32_t   text_len_max,
#                         bool   remove_special,
#                         bool   unparse_special);
@ctypes_function(
    "llama_detokenize",
    [
        llama_vocab_p_ctypes,
        llama_token_p,
        ctypes.c_int32,
        ctypes.c_char_p,
        ctypes.c_int32,
        ctypes.c_bool,
        ctypes.c_bool,
    ],
    ctypes.c_int32,
)
def llama_detokenize(
    vocab: llama_vocab_p,
    tokens: CtypesArray[llama_token],
    n_tokens: Union[ctypes.c_int, int],
    text: bytes,
    text_len_max: Union[ctypes.c_int, int],
    remove_special: Union[ctypes.c_bool, bool],
    unparse_special: Union[ctypes.c_bool, bool],
    /,
) -> int:
    """Convert the provided tokens into text (inverse of llama_tokenize()).

    Args:
        vocab: The model vocab to use for tokenization.
        tokens: The tokens to convert.
        n_tokens: The number of tokens.
        text: The buffer to write the text to.
        text_len_max: The length of the buffer.
        remove_special: Allow to remove BOS and EOS tokens if model is configured to do so.
        unparse_special: If true, special tokens are rendered in the output.
    """
    ...


# //
# // Chat templates
# //


# /// Apply chat template. Inspired by hf apply_chat_template() on python.
# ///
# /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
# /// @param tmpl A Jinja template to use for this chat.
# /// @param chat Pointer to a list of multiple llama_chat_message
# /// @param n_msg Number of llama_chat_message in this chat
# /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.
# /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)
# /// @param length The size of the allocated buffer
# /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.
# LLAMA_API int32_t llama_chat_apply_template(
#                         const char * tmpl,
#    const struct llama_chat_message * chat,
#                             size_t   n_msg,
#                               bool   add_ass,
#                               char * buf,
#                            int32_t   length);
@ctypes_function(
    "llama_chat_apply_template",
    [
        ctypes.c_char_p,  # tmpl
        ctypes.POINTER(llama_chat_message),  # chat
        ctypes.c_size_t,  # n_msg
        ctypes.c_bool,    # add_ass (added)
        ctypes.c_char_p,  # buf
        ctypes.c_int32,   # length
    ],
    ctypes.c_int32,
)
def llama_chat_apply_template(
    tmpl: bytes,
    chat: CtypesArray[llama_chat_message],
    n_msg: int,
    add_ass: bool,  # Added parameter
    buf: bytes,
    length: int,
    /,
) -> int:
    """Apply chat template.

    Args:
        tmpl: Template to use. If None, uses model's default
        chat: Array of chat messages
        n_msg: Number of messages
        add_ass: Whether to end prompt with assistant token
        buf: Output buffer
        length: Buffer length

    Returns:
        Number of bytes written, or needed if buffer too small
    """
    ...


# // Get list of built-in chat templates
# LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);
@ctypes_function(
    "llama_chat_builtin_templates",
    [
        ctypes.POINTER(ctypes.c_char_p),
        ctypes.c_size_t,
    ],
    ctypes.c_int32,
)
def llama_chat_builtin_templates(
    output: CtypesArray[ctypes.c_char_p],
    len: Union[ctypes.c_size_t, int],
    /,
) -> int:
    """Get list of built-in chat templates.

    Args:
        output: Output buffer to store template names.
        len: Length of the output buffer.

    Returns:
        Number of templates available.
        Returns a negative number on error.
    """
    ...


# //
# // Sampling API
# //
# // Sample usage:
# //
# //    // prepare the sampling chain at the start
# //    auto sparams = llama_sampler_chain_default_params();
# //
# //    llama_sampler * smpl = llama_sampler_chain_init(sparams);
# //
# //    llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50));
# //    llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1));
# //    llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8));
# //
# //    // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat"
# //    // this sampler will be responsible to select the actual token
# //    llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed));
# //
# //    ...
# //
# //    // decoding loop:
# //    while (...) {
# //        ...
# //
# //        llama_decode(ctx, batch);
# //
# //        // sample from the logits of the last token in the batch
# //        const llama_token id = llama_sampler_sample(smpl, ctx, -1);
# //
# //        ...
# //    }
# //
# //    llama_sampler_free(smpl);
# //

# typedef void * llama_sampler_context_t;
llama_sampler_context_t = ctypes.c_void_p


# struct llama_sampler_data {
#     struct ggml_tensor * logits;
#     struct ggml_tensor * probs;
#     struct ggml_tensor * sampled;
#     struct ggml_tensor * candidates;
# };
class llama_sampler_data(ctypes.Structure):
    if TYPE_CHECKING:
        logits: ctypes.c_void_p
        probs: ctypes.c_void_p
        sampled: ctypes.c_void_p
        candidates: ctypes.c_void_p

    _fields_ = [
        ("logits", ctypes.c_void_p),
        ("probs", ctypes.c_void_p),
        ("sampled", ctypes.c_void_p),
        ("candidates", ctypes.c_void_p),
    ]


# // user code can implement the interface below in order to create custom llama_sampler
# struct llama_sampler_i {
#     const char *           (*name)  (const struct llama_sampler * smpl);                                 // can be NULL
#     void                   (*accept)(      struct llama_sampler * smpl, llama_token token);              // can be NULL
#     void                   (*apply) (      struct llama_sampler * smpl, llama_token_data_array * cur_p); // required
#     void                   (*reset) (      struct llama_sampler * smpl);                                 // can be NULL
#     struct llama_sampler * (*clone) (const struct llama_sampler * smpl);                                 // can be NULL if ctx is NULL
#     void                   (*free)  (      struct llama_sampler * smpl);                                 // can be NULL if ctx is NULL

#     // [EXPERIMENTAL]
#     // backend sampling interface:

#     // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence
#     // note: call once per sampler
#     bool (*backend_init)(
#             struct llama_sampler       * smpl,
#             ggml_backend_buffer_type_t   buft,
#             uint32_t                     n_outputs_max_per_seq);

#     // call after .backend_apply()
#     void (*backend_accept)(
#             struct llama_sampler * smpl,
#             struct ggml_context  * ctx,
#             struct ggml_cgraph   * gf,
#             struct ggml_tensor   * selected_token);

#     // call after .backend_init()
#     void (*backend_apply)(
#             struct llama_sampler      * smpl,
#             struct ggml_context       * ctx,
#             struct ggml_cgraph        * gf,
#             struct llama_sampler_data * data);

#     // called before graph execution to set inputs for the current ubatch
#     void (*backend_set_input)(struct llama_sampler * smpl);

#     // called before rebuilding a sampling graph to clear any internal sampler state
#     void (*backend_reset)(struct llama_sampler * smpl);

#     // copy mutable state from src into dst while keeping dst's references to the current sampling graph
#     // src and dst must have the same type and configuration
#     void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst);
# };

# const char * (*name)(const struct llama_sampler * smpl);
llama_sampler_name_fn = ctypes.CFUNCTYPE(
    ctypes.c_char_p,    # return type
    ctypes.c_void_p     # smpl
)

# void (*accept)(struct llama_sampler * smpl, llama_token token);
llama_sampler_accept_fn = ctypes.CFUNCTYPE(
    None,                      # return void
    ctypes.c_void_p,           # smpl
    llama_token                # token
)

# void (*apply)(struct llama_sampler * smpl, llama_token_data_array * cur_p);
llama_sampler_apply_fn = ctypes.CFUNCTYPE(
    None,                                  # return void
    ctypes.c_void_p,                       # smpl
    ctypes.POINTER(llama_token_data_array) # cur_p
)

# void (*reset)(struct llama_sampler * smpl);
llama_sampler_reset_fn = ctypes.CFUNCTYPE(
    None,               # return void
    ctypes.c_void_p     # smpl
)

# struct llama_sampler * (*clone)(const struct llama_sampler * smpl);
llama_sampler_clone_fn = ctypes.CFUNCTYPE(
    ctypes.c_void_p,    # return struct llama_sampler *
    ctypes.c_void_p     # smpl (const ignored in ctypes)
)

# void (*free)(struct llama_sampler * smpl);
llama_sampler_free_fn = ctypes.CFUNCTYPE(
    None,               # return void
    ctypes.c_void_p     # smpl
)

# --- EXPERIMENTAL Backend Sampling Interface ---

# // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence
# // note: call once per sampler
# bool (*backend_init)(
#        struct llama_sampler       * smpl,
#        ggml_backend_buffer_type_t   buft,
#        uint32_t                     n_outputs_max_per_seq);
llama_sampler_backend_init_fn = ctypes.CFUNCTYPE(
    ctypes.c_bool,        # return bool
    ctypes.c_void_p,      # smpl
    ctypes.c_void_p,      # buft
    ctypes.c_uint32,      # n_outputs_max_per_seq
)

# // call after .backend_apply()
# void (*backend_accept)(struct llama_sampler * smpl, struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_tensor * selected_token);
llama_sampler_backend_accept_fn = ctypes.CFUNCTYPE(
    None,                  # return void
    ctypes.c_void_p,       # smpl
    ctypes.c_void_p,       # ctx
    ctypes.c_void_p,       # gf
    ctypes.c_void_p        # selected_token
)

# // call after .backend_init()
# void (*backend_apply)(struct llama_sampler * smpl, struct ggml_context * ctx, struct ggml_cgraph * gf, struct llama_sampler_data * data);
llama_sampler_backend_apply_fn = ctypes.CFUNCTYPE(
    None,                              # return void
    ctypes.c_void_p,                   # smpl
    ctypes.c_void_p,                   # ctx
    ctypes.c_void_p,                   # gf
    ctypes.POINTER(llama_sampler_data) # data
)

# // called before graph execution to set inputs for the current ubatch
# void (*backend_set_input)(struct llama_sampler * smpl);
llama_sampler_backend_set_input_fn = ctypes.CFUNCTYPE(
    None,              # return void
    ctypes.c_void_p    # smpl
)

# // called before rebuilding a sampling graph to clear any internal sampler state
# void (*backend_reset)(struct llama_sampler * smpl);
llama_sampler_backend_reset_fn = ctypes.CFUNCTYPE(
    None,              # return void
    ctypes.c_void_p    # smpl
)

# // copy mutable state from src into dst while keeping dst's references to the current sampling graph
# // src and dst must have the same type and configuration
# void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst);
llama_sampler_copy_state_fn = ctypes.CFUNCTYPE(
    None,                              # return void
    ctypes.c_void_p,                   # src
    ctypes.c_void_p,                   # dst
)

class llama_sampler_i(ctypes.Structure):
    _fields_ = [
        ("name",           llama_sampler_name_fn),
        ("accept",         llama_sampler_accept_fn),
        ("apply",          llama_sampler_apply_fn),
        ("reset",          llama_sampler_reset_fn),
        ("clone",          llama_sampler_clone_fn),
        ("free",           llama_sampler_free_fn),

        # [EXPERIMENTAL] Backend sampling interface
        ("backend_init",      llama_sampler_backend_init_fn),
        ("backend_accept",    llama_sampler_backend_accept_fn),
        ("backend_apply",     llama_sampler_backend_apply_fn),
        ("backend_set_input", llama_sampler_backend_set_input_fn),
        ("backend_reset",     llama_sampler_backend_reset_fn),
        ("copy_state",        llama_sampler_copy_state_fn),
    ]


# struct llama_sampler {
#     struct llama_sampler_i * iface;
#     llama_sampler_context_t  ctx;
# };
class llama_sampler(ctypes.Structure):
    _fields_ = [
        ("iface", ctypes.POINTER(llama_sampler_i)),
        ("ctx", llama_sampler_context_t),
    ]


if TYPE_CHECKING:
    llama_sampler_p = CtypesPointer[llama_sampler]

llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler)


# // [EXPERIMENTAL]
# // attach a sampler to the context
# // note: prefer initializing the context with llama_context_params.samplers when possible
# // note: changing the samplers of a context can cause graph reallocations and degraded performance
# LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl);
@ctypes_function(
    "llama_set_sampler",
    [llama_context_p_ctypes, llama_seq_id, llama_sampler_p_ctypes],
    ctypes.c_bool,
)
def llama_set_sampler(
    ctx: llama_context_p, seq_id: llama_seq_id, smpl: llama_sampler_p, /
) -> ctypes.c_bool:
    """
    attach a sampler to the context
    note: prefer initializing the context with llama_context_params.samplers when possible
    note: changing the samplers of a context can cause graph reallocations and degraded performance
    """
    ...


# // mirror of llama_sampler_i:
# LLAMA_API struct llama_sampler * llama_sampler_init  (     struct llama_sampler_i * iface, llama_sampler_context_t ctx);
@ctypes_function(
    "llama_sampler_init",
    [ctypes.POINTER(llama_sampler_i), llama_sampler_context_t],
    llama_sampler_p_ctypes,
)
def llama_sampler_init(
    iface: CtypesPointer[llama_sampler_i], ctx: llama_sampler_context_t, /
) -> llama_sampler_p:
    ...


# LLAMA_API const char *           llama_sampler_name  (const struct llama_sampler * smpl);
@ctypes_function(
    "llama_sampler_name",
    [llama_sampler_p_ctypes],
    ctypes.c_char_p,
)
def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes:
    ...


# LLAMA_API void                   llama_sampler_accept(      struct llama_sampler * smpl, llama_token token);
@ctypes_function(
    "llama_sampler_accept",
    [llama_sampler_p_ctypes, llama_token],
    None,
)
def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /):
    ...


# LLAMA_API void                   llama_sampler_apply (      struct llama_sampler * smpl, llama_token_data_array * cur_p);
@ctypes_function(
    "llama_sampler_apply",
    [llama_sampler_p_ctypes, llama_token_data_array_p],
    None,
)
def llama_sampler_apply(
    smpl: llama_sampler_p, cur_p: CtypesPointer[llama_token_data_array]):
    ...


# LLAMA_API void                   llama_sampler_reset (      struct llama_sampler * smpl);
@ctypes_function(
    "llama_sampler_reset",
    [llama_sampler_p_ctypes],
    None,
)
def llama_sampler_reset(smpl: llama_sampler_p, /):
    ...


# LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl);
@ctypes_function(
    "llama_sampler_clone",
    [llama_sampler_p_ctypes],
    llama_sampler_p_ctypes,
)
def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p:
    ...


# LLAMA_API void                   llama_sampler_copy  (const struct llama_sampler * src, struct llama_sampler * dst);
@ctypes_function(
    "llama_sampler_copy",
    [llama_sampler_p_ctypes, llama_sampler_p_ctypes],
    None,
)
def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p):
    ...


# // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)
# LLAMA_API void                   llama_sampler_free  (      struct llama_sampler * smpl);
@ctypes_function(
    "llama_sampler_free",
    [llama_sampler_p_ctypes],
    None,
)
def llama_sampler_free(smpl: llama_sampler_p, /):
    ...


# // llama_sampler_chain
# // a type of llama_sampler that can chain multiple samplers one after another
#
# LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params);
@ctypes_function(
    "llama_sampler_chain_init",
    [llama_sampler_chain_params],
    llama_sampler_p_ctypes,
)
def llama_sampler_chain_init(params: llama_sampler_chain_params, /) -> llama_sampler_p:
    ...


# // important: takes ownership of the sampler object and will free it when llama_sampler_free is called
# LLAMA_API void                   llama_sampler_chain_add(      struct llama_sampler * chain, struct llama_sampler * smpl);
@ctypes_function(
    "llama_sampler_chain_add",
    [llama_sampler_p_ctypes, llama_sampler_p_ctypes],
    None,
)
def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /):
    ...


# // return NULL if:
# //   - the sampler is NULL
# //   - the sampler is not a llama_sampler_chain
# //   - the index is out of bounds, unless i == -1
# //   - if i == -1, returns the chain itself (can be used to check if the sampler is a chain)
# LLAMA_API struct llama_sampler * llama_sampler_chain_get(      struct llama_sampler * chain, int32_t i);
@ctypes_function(
    "llama_sampler_chain_get",
    [llama_sampler_p_ctypes, ctypes.c_int32],
    llama_sampler_p_ctypes,
)
def llama_sampler_chain_get(
    chain: llama_sampler_p, i: Union[ctypes.c_int32, int], /
) -> llama_sampler_p:
    """
    return NULL if:
      - the sampler is NULL
      - the sampler is not a llama_sampler_chain
      - the index is out of bounds, unless i == -1
      - if i == -1, returns the chain itself (can be used to check if the sampler is a chain)
    """
    ...


# // the total number of samplers in the chain
# LLAMA_API int32_t                llama_sampler_chain_n  (const struct llama_sampler * chain);
@ctypes_function(
    "llama_sampler_chain_n",
    [llama_sampler_p_ctypes],
    ctypes.c_int32,
)
def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int:
    """
    the total number of samplers in the chain
    """
    ...


# // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed
# LLAMA_API struct llama_sampler * llama_sampler_chain_remove(   struct llama_sampler * chain, int32_t i);
@ctypes_function(
    "llama_sampler_chain_remove",
    [llama_sampler_p_ctypes, ctypes.c_int32],
    llama_sampler_p_ctypes,
)
def llama_sampler_chain_remove(
    chain: llama_sampler_p, i: Union[ctypes.c_int32, int], /
) -> llama_sampler_p:
    ...


# // available samplers:
#
# LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);
@ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes)
def llama_sampler_init_greedy() -> llama_sampler_p:
    ...


# /// seed == LLAMA_DEFAULT_SEED to use a random seed.
# LLAMA_API struct llama_sampler * llama_sampler_init_dist  (uint32_t seed);
@ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes)
def llama_sampler_init_dist(seed: int) -> llama_sampler_p:
    ...


# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
# /// Setting k <= 0 makes this a noop
# LLAMA_API struct llama_sampler * llama_sampler_init_top_k      (int32_t k);
@ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes)
def llama_sampler_init_top_k(k: int) -> llama_sampler_p:
    ...


# /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
# LLAMA_API struct llama_sampler * llama_sampler_init_top_p      (float   p, size_t min_keep);
@ctypes_function(
    "llama_sampler_init_top_p",
    [ctypes.c_float, ctypes.c_size_t],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p:
    ...


# /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841
# LLAMA_API struct llama_sampler * llama_sampler_init_min_p      (float   p, size_t min_keep);
@ctypes_function(
    "llama_sampler_init_min_p",
    [ctypes.c_float, ctypes.c_size_t],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p:
    ...


# /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
# LLAMA_API struct llama_sampler * llama_sampler_init_typical    (float   p, size_t min_keep);
@ctypes_function(
    "llama_sampler_init_typical",
    [ctypes.c_float, ctypes.c_size_t],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p:
    ...


# LLAMA_API struct llama_sampler * llama_sampler_init_temp       (float   t);
@ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes)
def llama_sampler_init_temp(t: float) -> llama_sampler_p:
    ...


# /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.
# LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext   (float   t, float   delta, float exponent);
@ctypes_function(
    "llama_sampler_init_temp_ext",
    [ctypes.c_float, ctypes.c_float, ctypes.c_float],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_temp_ext(
    t: float, delta: float, exponent: float
) -> llama_sampler_p:
    ...


# /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335
# LLAMA_API struct llama_sampler * llama_sampler_init_xtc        (float   p, float   t,     size_t min_keep, uint32_t seed);
@ctypes_function(
    "llama_sampler_init_xtc",
    [ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.c_uint32],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_xtc(
    p: float, t: float, min_keep: int, seed: int, /
) -> llama_sampler_p:
    ...


# /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641
# LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float   n);
@ctypes_function(
    "llama_sampler_init_top_n_sigma",
    [ctypes.c_float],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p:
    ...


# /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat(
#                          int32_t   n_vocab,
#                         uint32_t   seed,
#                            float   tau,
#                            float   eta,
#                          int32_t   m);
@ctypes_function(
    "llama_sampler_init_mirostat",
    [ctypes.c_int32, ctypes.c_uint32, ctypes.c_float, ctypes.c_float, ctypes.c_int32],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_mirostat(
    n_vocab: int, seed: int, tau: float, eta: float, m: int, /
) -> llama_sampler_p:
    ...


# /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
# /// @param tau  The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2(
#                         uint32_t   seed,
#                            float   tau,
#                            float   eta);
@ctypes_function(
    "llama_sampler_init_mirostat_v2",
    [ctypes.c_uint32, ctypes.c_float, ctypes.c_float],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_mirostat_v2(
    seed: int, tau: float, eta: float, /
) -> llama_sampler_p:
    ...


# /// @details Initializes a GBNF grammar, see grammars/README.md for details.
# /// @param vocab The vocabulary that this grammar will be used with.
# /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails.
# /// @param grammar_root The name of the start symbol for the grammar.
# LLAMA_API struct llama_sampler * llama_sampler_init_grammar(
#         const struct llama_vocab * vocab,
#                       const char * grammar_str,
#                       const char * grammar_root);
@ctypes_function(
    "llama_sampler_init_grammar",
    [llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_char_p],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar(
    vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, /
) -> llama_sampler_p:
    ...


# /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639
# /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group.
# /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included.
# LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(
#     const struct llama_vocab * vocab,
#                   const char * grammar_str,
#                   const char * grammar_root,
#                  const char ** trigger_patterns,
#                         size_t num_trigger_patterns,
#            const llama_token * trigger_tokens,
#                         size_t num_trigger_tokens);
@ctypes_function(
    "llama_sampler_init_grammar_lazy_patterns",
    [
        llama_vocab_p_ctypes,
        ctypes.c_char_p,
        ctypes.c_char_p,
        ctypes.POINTER(ctypes.c_char_p),
        ctypes.c_size_t,
        llama_token_p,
        ctypes.c_size_t,
    ],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_grammar_lazy_patterns(
    vocab: llama_vocab_p,
    grammar_str: bytes,
    grammar_root: bytes,
    trigger_patterns: CtypesArray[bytes], # type: ignore
    num_trigger_patterns: int,
    trigger_tokens: CtypesArray[llama_token],
    num_trigger_tokens: int,
    /,
) -> llama_sampler_p:
    ...


# /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
# LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
#                          int32_t   n_vocab,
#                          int32_t   penalty_last_n,   // last n tokens to penalize (0 = disable penalty)
#                            float   penalty_repeat,   // must be > 0.0, 1.0 = disabled
#                            float   penalty_freq,     // must be finite, 0.0 = disabled
#                            float   penalty_present); // must be finite, 0.0 = disabled
@ctypes_function(
    "llama_sampler_init_penalties",
    [
        ctypes.c_int32,
        ctypes.c_int32,
        ctypes.c_float,
        ctypes.c_float,
        ctypes.c_float,
    ],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_penalties(
    n_vocab: int,
    penalty_last_n: int,
    penalty_repeat: float,
    penalty_freq: float,
    penalty_present: float,
    /,
) -> llama_sampler_p:
    ...


# ///  @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
# LLAMA_API struct llama_sampler * llama_sampler_init_dry(
#         const struct llama_vocab *  vocab,
#                            float    dry_multiplier,
#                            float    dry_base,
#                          int32_t    dry_allowed_length,
#                          int32_t    dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty)
#                       const char ** seq_breakers,
#                           size_t    num_breakers);
@ctypes_function(
    "llama_sampler_init_dry",
    [
        llama_vocab_p_ctypes,
        ctypes.c_float,
        ctypes.c_float,
        ctypes.c_int32,
        ctypes.c_int32,
        ctypes.POINTER(ctypes.c_char_p),
        ctypes.c_size_t,
    ],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_dry(
    vocab: llama_vocab_p,
    dry_multiplier: float,
    dry_base: float,
    dry_allowed_length: int,
    dry_penalty_last_n: int,
    seq_breakers: CtypesArray[ctypes.c_char_p],
    num_breakers: int,
    /,
) -> llama_sampler_p:
    ...


# /// adaptive-p: select tokens near a configurable target probability over time.
# ///
# /// the adaptive-p sampler transforms the token probability distribution to favor tokens
# /// that fall near a user-configurable probability target.
# ///
# /// internally, the sampler maintains an exponential moving average of the *ORIGINAL*
# /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an
# /// adapted target probability at each sampling step, thus maintaining the desired target
# /// probability over time.
# ///
# /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last
# /// in the sampler chain (like mirostat, dist, greedy).
# ///
# /// only mild truncation before this sampler is recommended. we suggest applying min-p
# /// before adaptive-p as the only other active sampler in the chain.
# ///
# /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)
# /// @param decay  EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99)
# /// @param seed   RNG seed
# ///
# /// ref: https://github.com/ggml-org/llama.cpp/pull/17927
# ///
# LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p(
#                             float   target,
#                             float   decay,
#                         uint32_t   seed);
@ctypes_function(
    "llama_sampler_init_adaptive_p",
    [
        ctypes.c_float,
        ctypes.c_float,
        ctypes.c_uint32,
    ],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_adaptive_p(
    target: float,
    decay: float,
    seed: int,
    /,
) -> llama_sampler_p:
    ...


# LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias(
#                          int32_t   n_vocab,
#                          int32_t   n_logit_bias,
#           const llama_logit_bias * logit_bias);
@ctypes_function(
    "llama_sampler_init_logit_bias",
    [ctypes.c_int32, ctypes.c_int32, llama_logit_bias_p],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_logit_bias(
    n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], /
) -> llama_sampler_p:
    ...


# // this sampler is meant to be used for fill-in-the-middle infilling
# // it's supposed to be used after top_k + top_p sampling
# //
# // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG
# // 2. combine probs of tokens that have the same prefix
# //
# // example:
# //
# // - before:
# //   "hel":   0.5
# //   "hell":  0.2
# //   "hello": 0.1
# //   "dummy": 0.1
# //
# // - after:
# //   "hel":   0.8
# //   "dummy": 0.1
# //
# // 3. discard non-EOG tokens with low prob
# // 4. if no tokens are left -> pick EOT
# //
# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab);
@ctypes_function(
    "llama_sampler_init_infill",
    [llama_vocab_p_ctypes],
    llama_sampler_p_ctypes,
)
def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p:
    ...


# // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise
# LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl);
@ctypes_function(
    "llama_sampler_get_seed",
    [llama_sampler_p_ctypes],
    ctypes.c_uint32,
)
def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int:
    ...


# /// @details Sample and accept a token from the idx-th output of the last evaluation
# // For multiple outputs from one sampler, call this function in output order without gaps.
# //
# // Shorthand for:
# //    const auto * logits = llama_get_logits_ith(ctx, idx);
# //    llama_token_data_array cur_p = { ... init from logits ... };
# //    llama_sampler_apply(smpl, &cur_p);
# //    auto token = cur_p.data[cur_p.selected].id;
# //    llama_sampler_accept(smpl, token);
# //    return token;
# // Returns the sampled token
# LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx);
@ctypes_function(
    "llama_sampler_sample",
    [llama_sampler_p_ctypes, llama_context_p_ctypes, ctypes.c_int32],
    llama_token,
)
def llama_sampler_sample(
    smpl: llama_sampler_p, ctx: llama_context_p, idx: ctypes.c_int32, /
) -> ctypes.c_int32:
    ...


# //
# // Model split
# //


# /// @details Build a split GGUF final path for this chunk.
# ///          llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf"
# //  Returns the split_path length.
# LLAMA_API int32_t  llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count);
@ctypes_function(
    "llama_split_path",
    [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int32, ctypes.c_int32],
    ctypes.c_int32,
)
def llama_split_path(
    split_path: bytes,
    maxlen: Union[ctypes.c_size_t, int],
    path_prefix: bytes,
    split_no: Union[ctypes.c_int32, int],
    split_count: Union[ctypes.c_int32, int],
    /,
) -> int:
    """Build a split GGUF final path for this chunk."""
    ...


# /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match.
# ///          llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0"
# //  Returns the split_prefix length.
# LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count);
@ctypes_function(
    "llama_split_prefix",
    [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int32, ctypes.c_int32],
    ctypes.c_int32,
)
def llama_split_prefix(
    split_prefix: bytes,
    maxlen: Union[ctypes.c_size_t, int],
    split_path: bytes,
    split_no: Union[ctypes.c_int32, int],
    split_count: Union[ctypes.c_int32, int],
    /,
) -> int:
    """Extract the path prefix from the split_path if and only if the split_no and split_count match."""
    ...


# // Print system information
# LLAMA_API const char * llama_print_system_info(void);
@ctypes_function("llama_print_system_info", [], ctypes.c_char_p)
def llama_print_system_info() -> bytes:
    ...


# // Set callback for all future logging events.
# // If this is not called, or NULL is supplied, everything is output on stderr.
# // The logger state is global so these functions are NOT thread safe.
# LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data);
@ctypes_function(
    "llama_log_get",
    [ctypes.POINTER(ggml_log_callback), ctypes.POINTER(ctypes.c_void_p)],
    None,
)
def llama_log_get(
    log_callback: Optional[ctypes.pointer(ggml_log_callback)], # type: ignore
    user_data: ctypes.pointer(ctypes.c_void_p), # type: ignore
    /,
):
    """Get callback for all future logging events.
    If this is not called, or NULL is supplied, everything is output on stderr."""
    ...


# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
@ctypes_function(
    "llama_log_set",
    [ggml_log_callback, ctypes.c_void_p],
    None,
)
def llama_log_set(
    log_callback: Optional[ggml_log_callback], # type: ignore
    user_data: ctypes.c_void_p,
    /,
):
    """Set callback for all future logging events.
    If this is not called, or NULL is supplied, everything is output on stderr."""
    ...


# //
# // Performance utils
# //
# // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements.
# //


# struct llama_perf_context_data {
#     // ms == milliseconds
#     double t_start_ms;  // absolute start time
#     double t_load_ms;   // time needed for loading the model
#     double t_p_eval_ms; // time needed for processing the prompt
#     double t_eval_ms;   // time needed for generating tokens

#     int32_t n_p_eval;   // number of prompt tokens
#     int32_t n_eval;     // number of generated tokens
#     int32_t n_reused;   // number of times a ggml compute graph had been reused
#     int32_t n_reallocs_same; // number of graph reallocations during inference with an unchanged graph size
#     int32_t n_reallocs_diff; // number of graph reallocations during inference with a changed graph size
# };
class llama_perf_context_data(ctypes.Structure):
    """Timing interval and graph statistics returned by llama.cpp.

    n_reallocs_same/diff count compute-buffer reallocations with unchanged/
    changed graph size, excluding changes in backend assignments. They are
    context lifetime totals and survive llama_perf_context_reset().
    """
    _fields_ = [
        ("t_start_ms", ctypes.c_double),
        ("t_load_ms", ctypes.c_double),
        ("t_p_eval_ms", ctypes.c_double),
        ("t_eval_ms", ctypes.c_double),
        ("n_p_eval", ctypes.c_int32),
        ("n_eval", ctypes.c_int32),
        ("n_reused", ctypes.c_int32),
        ("n_reallocs_same", ctypes.c_int32),
        ("n_reallocs_diff", ctypes.c_int32),
    ]


# struct llama_perf_sampler_data {
#     double t_sample_ms; // time needed for sampling in ms
#     int32_t n_sample;   // number of sampled tokens
# };
class llama_perf_sampler_data(ctypes.Structure):
    _fields_ = [
        ("t_sample_ms", ctypes.c_double),
        ("n_sample", ctypes.c_int32),
    ]


# LLAMA_API struct llama_perf_context_data llama_perf_context      (const struct llama_context * ctx);
@ctypes_function(
    "llama_perf_context",
    [llama_context_p_ctypes],
    llama_perf_context_data,
)
def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data:
    """Read interval statistics and lifetime graph-reallocation counters."""
    ...


# LLAMA_API void                           llama_perf_context_print(const struct llama_context * ctx);
@ctypes_function(
    "llama_perf_context_print",
    [llama_context_p_ctypes],
    None,
)
def llama_perf_context_print(ctx: llama_context_p, /):
    ...


# LLAMA_API void                           llama_perf_context_reset(      struct llama_context * ctx);
@ctypes_function(
    "llama_perf_context_reset",
    [llama_context_p_ctypes],
    None,
)
def llama_perf_context_reset(ctx: llama_context_p, /):
    """Reset the timing/reuse interval; preserve load time and reallocation totals."""
    ...


# // NOTE: the following work only with samplers constructed via llama_sampler_chain_init
# LLAMA_API struct llama_perf_sampler_data llama_perf_sampler      (const struct llama_sampler * chain);
@ctypes_function(
    "llama_perf_sampler",
    [llama_sampler_p_ctypes],
    llama_perf_sampler_data,
)
def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data:
    ...


# LLAMA_API void                           llama_perf_sampler_print(const struct llama_sampler * chain);
@ctypes_function(
    "llama_perf_sampler_print",
    [llama_sampler_p_ctypes],
    None,
)
def llama_perf_sampler_print(chain: llama_sampler_p, /):
    ...


# LLAMA_API void                           llama_perf_sampler_reset(      struct llama_sampler * chain);
@ctypes_function(
    "llama_perf_sampler_reset",
    [llama_sampler_p_ctypes],
    None,
)
def llama_perf_sampler_reset(chain: llama_sampler_p, /):
    ...


# //
# // training
# //

# // function that returns whether or not a given tensor contains trainable parameters
# typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata);
llama_opt_param_filter = ctypes.CFUNCTYPE(
    ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p
)


# // always returns true
# LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata);
@ctypes_function("llama_opt_param_filter_all", [ctypes.c_void_p, ctypes.c_void_p], ctypes.c_bool)
def llama_opt_param_filter_all(
    tensor: llama_model_p,
    userdata: ctypes.c_void_p, /
) -> bool:
    ...

# struct llama_opt_params {
#     uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0
#     llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters
#     void * param_filter_ud;              // userdata for determining which tensors contain trainable parameters
#     ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters
#     void * get_opt_pars_ud;                     // userdata for calculating optimizer parameters
#     enum ggml_opt_optimizer_type optimizer_type;
# };
class llama_opt_params(ctypes.Structure):
    _fields_ = [
        ("n_ctx_train", ctypes.c_uint32),
        ("param_filter", llama_opt_param_filter),
        ("param_filter_ud", ctypes.c_void_p),
        ("get_opt_pars", ggml_opt_get_optimizer_params),
        ("get_opt_pars_ud", ctypes.c_void_p),
        ("optimizer_type", ctypes.c_int),
    ]


# LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
@ctypes_function(
    "llama_opt_init",
    [llama_context_p_ctypes, llama_model_p_ctypes, llama_opt_params],
    None,
)
def llama_opt_init(
    lctx: llama_context_p,
    model: llama_model_p,
    lopt_params: llama_opt_params, /
):
    ...

# LLAMA_API void llama_opt_epoch(
#         struct llama_context    * lctx,
#         ggml_opt_dataset_t        dataset,
#         ggml_opt_result_t         result_train,
#         ggml_opt_result_t         result_eval,
#         int64_t                   idata_split,
#         ggml_opt_epoch_callback   callback_train,
#         ggml_opt_epoch_callback   callback_eval);
@ctypes_function(
    "llama_opt_epoch",[
        llama_context_p_ctypes,
        ctypes.c_void_p,
        ctypes.c_void_p,
        ctypes.c_void_p,
        ctypes.c_int64,
        ctypes.c_void_p,
        ctypes.c_void_p
    ],
    None,
)
def llama_opt_epoch(
    lctx: llama_context_p,
    dataset: ctypes.c_void_p,
    result_train: ctypes.c_void_p,
    result_eval: ctypes.c_void_p,
    idata_split: ctypes.c_int64,
    callback_train: ctypes.c_void_p,
    callback_eval: ctypes.c_void_p, /
):
    ...

##############################
# // llama.cpp/src/llama-ext.h
##############################

# // this is a staging header for new llama.cpp API
# // breaking changes and C++ are allowed. everything here should be considered WIP
# // try as much as possible to not include this header in the rest of the codebase

# Optional bindings accept C, MSVC, and Itanium ABI symbol names. Missing
# symbols raise RuntimeError when called, so older libraries remain importable.
ctypes_function_llama_ext = ctypes_function_for_shared_library(_lib)

# // Reserve a new compute graph. It is valid until the next call to llama_graph_reserve.
# LLAMA_API struct ggml_cgraph * llama_graph_reserve(
#         struct llama_context * ctx,
#         uint32_t n_tokens,
#         uint32_t n_seqs,
#         uint32_t n_outputs);
@ctypes_function_llama_ext(
    [
        "llama_graph_reserve",
        "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z",
        "__Z19llama_graph_reserveP13llama_contextjjj",
        "_Z19llama_graph_reserveP13llama_contextjjj",
    ],
    [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32],
    ctypes.POINTER(ggml_cgraph),
    required=False,
)
def llama_graph_reserve(
    ctx: llama_context_p,
    n_tokens: ctypes.c_uint32,
    n_seqs: ctypes.c_uint32,
    n_outputs: ctypes.c_uint32,
) -> ctypes.POINTER(ggml_cgraph):  # type: ignore
    """Reserve a compute graph for the specified batch dimensions.

    Args:
        ctx: Context that owns the graph.
        n_tokens: Number of input tokens.
        n_seqs: Number of sequences.
        n_outputs: Number of requested output rows.

    Returns:
        A borrowed graph pointer, valid until the next graph reservation.
        Do not free it separately from the context.
    """
    ...

# // Get the default ggml_type for a given ftype.
# LLAMA_API ggml_type llama_ftype_get_default_type(llama_ftype ftype);
@ctypes_function_llama_ext(
    [
        "llama_ftype_get_default_type",
        "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z",
        "__Z28llama_ftype_get_default_type11llama_ftype",
        "_Z28llama_ftype_get_default_type11llama_ftype",
    ],
    [ctypes.c_int],
    int,
    required=False,
)
def llama_ftype_get_default_type(
    ftype: llama_ftype
) -> int:
    """Return the default ggml_type for a llama_ftype value.

    Individual tensors may receive a different type during quantization.
    """
    ...

# struct quantize_state_impl;
quantize_state_impl_p = NewType("quantize_state_impl_p", int)
quantize_state_impl_p_ctypes = ctypes.c_void_p


# LLAMA_API quantize_state_impl * llama_quant_init(
#         const llama_model * model,
#         const llama_model_quantize_params * params);
@ctypes_function_llama_ext(
    [
        "llama_quant_init",
        "?llama_quant_init@@YAPEAUquantize_state_impl@@PEBUllama_model@@PEBUllama_model_quantize_params@@@Z",
        "__Z16llama_quant_initPK11llama_modelPK27llama_model_quantize_params",
        "_Z16llama_quant_initPK11llama_modelPK27llama_model_quantize_params",
    ],
    [llama_model_p_ctypes, ctypes.POINTER(llama_model_quantize_params)],
    quantize_state_impl_p_ctypes,
    required=False,
)
def llama_quant_init(
    model: llama_model_p,
    params: CtypesPointerOrRef[llama_model_quantize_params],
) -> quantize_state_impl_p:
    """Create quantization state; release it with llama_quant_free().

    Args:
        model: Model whose architecture and hyperparameters guide quantization.
        params: Pointer to the quantization parameters.

    Returns:
        A caller-owned opaque state pointer.

    Keep the model and the parameters (including their pointer-backed data)
    alive until the state is freed.
    """
    ...


# LLAMA_API void llama_quant_free(quantize_state_impl * qs);
@ctypes_function_llama_ext(
    [
        "llama_quant_free",
        "?llama_quant_free@@YAXPEAUquantize_state_impl@@@Z",
        "__Z16llama_quant_freeP19quantize_state_impl",
        "_Z16llama_quant_freeP19quantize_state_impl",
    ],
    [quantize_state_impl_p_ctypes],
    None,
    required=False,
)
def llama_quant_free(qs: quantize_state_impl_p):
    """Release quantization state created by llama_quant_init().

    The model and parameter storage are not freed. Do not reuse qs afterward.
    """
    ...


class llama_quant_model_desc(ctypes.Structure):
    """Metadata descriptor for a mock model used in quantization tests."""

    _fields_ = [
        ("architecture", ctypes.c_char_p),
        ("n_embd", ctypes.c_uint32),
        ("n_ff", ctypes.c_uint32),
        ("n_layer", ctypes.c_uint32),
        ("n_head", ctypes.c_uint32),
        ("n_head_kv", ctypes.c_uint32),
        ("n_expert", ctypes.c_uint32),
        ("n_embd_head_k", ctypes.c_uint32),
        ("n_embd_head_v", ctypes.c_uint32),
    ]


# LLAMA_API llama_model * llama_quant_model_from_metadata(const llama_quant_model_desc * desc);
@ctypes_function_llama_ext(
    [
        "llama_quant_model_from_metadata",
        "?llama_quant_model_from_metadata@@YAPEAUllama_model@@PEBUllama_quant_model_desc@@@Z",
        "__Z31llama_quant_model_from_metadataPK22llama_quant_model_desc",
        "_Z31llama_quant_model_from_metadataPK22llama_quant_model_desc",
    ],
    [ctypes.POINTER(llama_quant_model_desc)],
    llama_model_p_ctypes,
    required=False,
)
def llama_quant_model_from_metadata(
    desc: CtypesPointerOrRef[llama_quant_model_desc],
) -> llama_model_p:
    """Create a mock model from a llama_quant_model_desc pointer.

    The model supplies metadata for quantization planning, not loaded weights
    for inference. Release any dependent quantization state before calling
    llama_model_free() on the returned model.
    """
    ...


# LLAMA_API bool llama_quant_tensor_allows_quantization(
#         const quantize_state_impl * qs,
#         const ggml_tensor * tensor);
@ctypes_function_llama_ext(
    [
        "llama_quant_tensor_allows_quantization",
        "?llama_quant_tensor_allows_quantization@@YA_NPEBUquantize_state_impl@@PEBUggml_tensor@@@Z",
        "__Z38llama_quant_tensor_allows_quantizationPK19quantize_state_implPK11ggml_tensor",
        "_Z38llama_quant_tensor_allows_quantizationPK19quantize_state_implPK11ggml_tensor",
    ],
    [quantize_state_impl_p_ctypes, ggml_tensor_p],
    ctypes.c_bool,
    required=False,
)
def llama_quant_tensor_allows_quantization(
    qs: quantize_state_impl_p,
    tensor: ggml_tensor_p,     # type: ignore
) -> bool:
    """Return whether a tensor can be quantized with this state's parameters.

    Pass a live quantization state and a valid ggml_tensor pointer. The native
    check uses the tensor's name, dimensions, and the model architecture.
    """
    ...


# LLAMA_API void llama_quant_compute_types(
#         quantize_state_impl * qs,
#         llama_ftype ftype,
#         ggml_tensor ** tensors,
#         ggml_type * result_types,
#         size_t n_tensors);
@ctypes_function_llama_ext(
    [
        "llama_quant_compute_types",
        "?llama_quant_compute_types@@YAXPEAUquantize_state_impl@@W4llama_ftype@@PEAPEAUggml_tensor@@PEAW4ggml_type@@_K@Z",
        "__Z25llama_quant_compute_typesP19quantize_state_impl11llama_ftypePP11ggml_tensorP9ggml_typem",
        "_Z25llama_quant_compute_typesP19quantize_state_impl11llama_ftypePP11ggml_tensorP9ggml_typem",
    ],
    [
        quantize_state_impl_p_ctypes,
        ctypes.c_int,
        ctypes.POINTER(ggml_tensor_p),
        ctypes.POINTER(ctypes.c_int),
        ctypes.c_size_t,
    ],
    None,
    required=False,
)
def llama_quant_compute_types(
    qs: quantize_state_impl_p,
    ftype: int,
    tensors: CtypesArray[ggml_tensor_p],     # type: ignore
    result_types: CtypesArray[ctypes.c_int],
    n_tensors: int,
):
    """Fill a caller-owned ggml_type array with quantization assignments.

    Args:
        qs: Live quantization state.
        ftype: Target llama_ftype for this computation.
        tensors: Array of ggml_tensor pointers to classify.
        result_types: Writable array of C int enum values, one per tensor.
        n_tensors: Number of entries to read and write.

    Both arrays must contain at least n_tensors entries. Every tensor must
    pass llama_quant_tensor_allows_quantization() before this call.
    """
    ...


# LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model);
@ctypes_function_llama_ext(
    [
        "llama_model_n_expert",
        "?llama_model_n_expert@@YAHPEBUllama_model@@@Z",
        "__Z20llama_model_n_expertPK11llama_model",
        "_Z20llama_model_n_expertPK11llama_model",
    ],
    [llama_model_p_ctypes],
    ctypes.c_int32,
    required=False,
)
def llama_model_n_expert(
    model: llama_model_p
) -> ctypes.c_int32:
    """Return the model's expert count, or zero for a model without experts."""
    ...

# LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model);
@ctypes_function_llama_ext(
    [
        "llama_model_n_devices",
        "?llama_model_n_devices@@YAHPEBUllama_model@@@Z",
        "__Z21llama_model_n_devicesPK11llama_model",
        "_Z21llama_model_n_devicesPK11llama_model",
    ],
    [llama_model_p_ctypes],
    ctypes.c_int32,
    required=False,
)
def llama_model_n_devices(
    model: llama_model_p
) -> ctypes.c_int32:
    """Return the number of device entries assigned to this model."""
    ...

# LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i);
@ctypes_function_llama_ext(
    [
        "llama_model_get_device",
        "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z",
        "__Z22llama_model_get_devicePK11llama_modeli",
        "_Z22llama_model_get_devicePK11llama_modeli",
    ],
    [llama_model_p_ctypes, ctypes.c_int],
    ctypes.c_void_p,
    required=False,
)
def llama_model_get_device(
    model: llama_model_p,
    i: int,
) -> ctypes.c_void_p:
    """Return the borrowed backend device handle at zero-based index i.

    Valid indices are below llama_model_n_devices(model). An out-of-range
    index returns NULL (None); ownership of the device is not transferred.
    """
    ...

# LLAMA_API void llama_model_set_attn_rot(struct llama_model * model, bool value);
@ctypes_function_llama_ext(
    [
        "llama_model_set_attn_rot",
        "?llama_model_set_attn_rot@@YAXPEAUllama_model@@_N@Z",
        "__Z24llama_model_set_attn_rotP11llama_modelb",
        "_Z24llama_model_set_attn_rotP11llama_modelb",
    ],
    [llama_model_p_ctypes, ctypes.c_bool],
    None,
    required=False,
)
def llama_model_set_attn_rot(model: llama_model_p, value: bool) -> None:
    """Toggle KV attention rotation for contexts subsequently created from this model."""
    ...


# // Set whether the context outputs nextn embeddings or not
# // If masked == true,  output the embeddings only for the tokens with batch.logits != 0
# // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits
# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked);
@ctypes_function_llama_ext(
    [
        "llama_set_embeddings_nextn",
        "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z",
        "__Z26llama_set_embeddings_nextnP13llama_contextbb",
        "_Z26llama_set_embeddings_nextnP13llama_contextbb",
    ],
    [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool],
    None,
    required=False,
)
def llama_set_embeddings_nextn(
    ctx: llama_context_p,
    value: bool,
    masked: bool,
):
    """Enable or disable NextN embeddings used by speculative decoding.

    Args:
        ctx: Context whose output configuration is changed.
        value: Whether to produce NextN embeddings.
        masked: If true, emit only rows selected by batch output flags.
            Otherwise, emit a row for every input token.
    """
    ...

# // Select which appended NextN block the DECODER_MTP graph runs (offset past
# // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to
# // chain multiple trained NextN heads. Default 0 (first head).
# LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);
@ctypes_function_llama_ext(
    [
        "llama_set_nextn_layer_offset",
        "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z",
        "__Z28llama_set_nextn_layer_offsetP13llama_contexti",
        "_Z28llama_set_nextn_layer_offsetP13llama_contexti",
    ],
    [llama_context_p_ctypes, ctypes.c_int32],
    None,
    required=False,
)
def llama_set_nextn_layer_offset(
    ctx: llama_context_p,
    offset: ctypes.c_int32,
):
    """Select the appended NextN block used by the DECODER_MTP graph.

    offset is zero-based: the selected layer is n_layer() + offset. Use a
    valid trained head index; zero selects the first head and is the default.
    """
    ...

# // Marks the entries that a joint decision head (clef) reads, the default is 0
# // See https://github.com/ggml-org/llama.cpp/pull/29831 for details
# // A run of entries with the same value is one span, spans must be separated by entries with value 0
# // An option belongs to the last question before it
# enum llama_decision_order {
#     LLAMA_DECISION_ORDER_NONE            = 0, // not read by the head
#     LLAMA_DECISION_ORDER_QUESTION_NOUL   = 1, // text of a question
#     LLAMA_DECISION_ORDER_QUESTION_CHOICE = 2,
#     LLAMA_DECISION_ORDER_QUESTION_SCORE  = 3,
#     LLAMA_DECISION_ORDER_OPTION          = 4, // text of an option
# };
class llama_decision_order(enum.IntEnum):
    LLAMA_DECISION_ORDER_NONE            = 0  # not read by the head
    LLAMA_DECISION_ORDER_QUESTION_NOUL   = 1  # text of a question
    LLAMA_DECISION_ORDER_QUESTION_CHOICE = 2
    LLAMA_DECISION_ORDER_QUESTION_SCORE  = 3
    LLAMA_DECISION_ORDER_OPTION          = 4  # text of an option

# // The embeddings output has one value per entry: row i is the score of option i
# LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
@ctypes_function_llama_ext(
    [
        "llama_batch_ext_set_decision_order",
        "?llama_batch_ext_set_decision_order@@YA_NPEAUllama_batch_ext@@HW4llama_decision_order@@@Z",
        "__Z34llama_batch_ext_set_decision_orderP15llama_batch_exti20llama_decision_order",
        "_Z34llama_batch_ext_set_decision_orderP15llama_batch_exti20llama_decision_order",
    ],
    [llama_batch_ext_p_ctypes, ctypes.c_int32, ctypes.c_int],
    ctypes.c_bool,
    required=False,
)
def llama_batch_ext_set_decision_order(
    batch: llama_batch_ext_p,
    idx: ctypes.c_int32,
    order: int,
) -> bool:
    """Mark a batch entry as question text, option text, or ordinary text.

    Args:
        batch: Extended batch containing the entry.
        idx: Zero-based entry index.
        order: A llama_decision_order value.

    Returns:
        Whether the native batch accepted the update.

    Consecutive entries with the same nonzero order form a span. Separate
    spans with order NONE; each option belongs to the preceding question.
    The joint decision head returns option scores through embedding rows.
    """
    ...

# // mirrors:
# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx);
@ctypes_function_llama_ext(
    [
        "llama_get_embeddings_nextn",
        "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z",
        "__Z26llama_get_embeddings_nextnP13llama_context",
        "_Z26llama_get_embeddings_nextnP13llama_context",
    ],
    [llama_context_p_ctypes],
    ctypes.POINTER(ctypes.c_float),
    required=False,
)
def llama_get_embeddings_nextn(
    ctx: llama_context_p,
) -> ctypes.POINTER(ctypes.c_float):  # type: ignore
    """Return the contiguous NextN embedding output from the last evaluation.

    Enable it with llama_set_embeddings_nextn() before evaluation. Rows have
    llama_model_n_embd_out() floats. The context owns the buffer; copy values
    before another evaluation or before freeing the context.
    """
    ...

# // LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i);
@ctypes_function_llama_ext(
    [
        "llama_get_embeddings_nextn_ith",
        "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z",
        "__Z30llama_get_embeddings_nextn_ithP13llama_contexti",
        "_Z30llama_get_embeddings_nextn_ithP13llama_contexti",
    ],
    [llama_context_p_ctypes, ctypes.c_int32],
    ctypes.POINTER(ctypes.c_float),
    required=False,
)
def llama_get_embeddings_nextn_ith(
    ctx: llama_context_p,
    i: ctypes.c_int32,
) -> ctypes.POINTER(ctypes.c_float):  # type: ignore
    """Return a borrowed pointer to one NextN embedding row.

    With masked output, i follows llama_get_embeddings_ith() indexing.
    With unmasked output, i is a nonnegative input-token row index.
    Read llama_model_n_embd_out() floats and copy them before reevaluation.
    """
    ...

# // Set whether the context outputs the input embeddings of a specific layer
# LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value);
@ctypes_function_llama_ext(
    [
        "llama_set_embeddings_layer_inp",
        "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z",
        "__Z30llama_set_embeddings_layer_inpP13llama_contextjb",
        "_Z30llama_set_embeddings_layer_inpP13llama_contextjb",
    ],
    [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_bool],
    None,
    required=False,
)
def llama_set_embeddings_layer_inp(
    ctx: llama_context_p,
    lid: ctypes.c_uint32,
    value: bool,
) -> None:  # type: ignore
    """Enable or disable capture of input embeddings for layer lid.

    Set this before evaluation, using a valid zero-based model layer index.
    DFlash uses these intermediate embeddings to condition its draft model.
    """
    ...

# // mirrors:
# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
# LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid);
@ctypes_function_llama_ext(
    [
        "llama_get_embeddings_layer_inp",
        "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z",
        "__Z30llama_get_embeddings_layer_inpP13llama_contextj",
        "_Z30llama_get_embeddings_layer_inpP13llama_contextj",
    ],
    [llama_context_p_ctypes, ctypes.c_uint32],
    ctypes.POINTER(ctypes.c_float),
    required=False,
)
def llama_get_embeddings_layer_inp(
    ctx: llama_context_p,
    lid: ctypes.c_uint32,
) -> ctypes.POINTER(ctypes.c_float):  # type: ignore
    """Return captured input embeddings for zero-based layer lid.

    Capture must be enabled and the layer evaluated before this call; the
    native implementation asserts that the output exists. The context owns
    the buffer, which must be copied before another evaluation or teardown.
    """
    ...

# LLAMA_API void llama_set_embeddings_layer_inp_dev(struct llama_context * ctx, bool value);
@ctypes_function_llama_ext(
    [
        "llama_set_embeddings_layer_inp_dev",
        "?llama_set_embeddings_layer_inp_dev@@YAXPEAUllama_context@@_N@Z",
        "__Z34llama_set_embeddings_layer_inp_devP13llama_contextb",
        "_Z34llama_set_embeddings_layer_inp_devP13llama_contextb",
    ],
    [llama_context_p_ctypes, ctypes.c_bool],
    None,
    required=False,
)
def llama_set_embeddings_layer_inp_dev(ctx: llama_context_p, value: bool) -> None:
    """Capture enabled layer inputs on the device instead of in host buffers.

    Rows follow the original input batch order, with layers concatenated in
    ascending layer-ID order. Configure before decoding. The context owns the
    buffer, which is reused by subsequent evaluations.
    """
    ...


# LLAMA_API bool llama_supports_embd_row(const struct llama_context * ctx);
@ctypes_function_llama_ext(
    [
        "llama_supports_embd_row",
        "?llama_supports_embd_row@@YA_NPEBUllama_context@@@Z",
        "__Z23llama_supports_embd_rowPK13llama_context",
        "_Z23llama_supports_embd_rowPK13llama_context",
    ],
    [llama_context_p_ctypes],
    ctypes.c_bool,
    required=False,
)
def llama_supports_embd_row(ctx: llama_context_p) -> bool:
    """Whether this DFlash/DSpark context can read ctx_other's device features.

    Enable device layer-input capture on ctx_other before querying. A backend
    must support reading that buffer for this to return True.
    """
    ...


# LLAMA_API int32_t llama_batch_ext_add_embd_row(struct llama_batch_ext * batch, llama_seq_id seq_id, int32_t row);
@ctypes_function_llama_ext(
    [
        "llama_batch_ext_add_embd_row",
        "?llama_batch_ext_add_embd_row@@YAHPEAUllama_batch_ext@@HH@Z",
        "__Z28llama_batch_ext_add_embd_rowP15llama_batch_extii",
        "_Z28llama_batch_ext_add_embd_rowP15llama_batch_extii",
    ],
    [llama_batch_ext_p_ctypes, llama_seq_id, ctypes.c_int32],
    ctypes.c_int32,
    required=False,
)
def llama_batch_ext_add_embd_row(batch: llama_batch_ext_p, seq_id: int, row: int) -> int:
    """Append a reference to a row of ctx_other's layer-input device buffer.

    Set positions with llama_batch_ext_set_pos(). All submitted entries must
    reference device rows, without token IDs or host embeddings. Returns the
    batch index, or a negative add error code. Consume before ctx_other reuses
    its buffer; this call does not copy or own the source features.
    """
    ...


# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx);
@ctypes_function_llama_ext(
    [
        "llama_get_ctx_other",
        "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z",
        "__Z19llama_get_ctx_otherP13llama_context",
        "_Z19llama_get_ctx_otherP13llama_context",
    ],
    [llama_context_p_ctypes],
    llama_context_p_ctypes,
    required=False,
)
def llama_get_ctx_other(
    ctx: llama_context_p,
) -> llama_context_p:
    """Return the related context configured through ctx_other, or NULL.

    This is a borrowed handle used by paired speculative contexts. This call
    does not create a context or transfer responsibility for freeing it.
    """
    ...

# // model/context data extraction

# LLAMA_API int32_t llama_model_dflash_selector_top_k(const struct llama_model * model);
@ctypes_function_llama_ext(
    [
        "llama_model_dflash_selector_top_k",
        "?llama_model_dflash_selector_top_k@@YAHPEBUllama_model@@@Z",
        "__Z33llama_model_dflash_selector_top_kPK11llama_model",
        "_Z33llama_model_dflash_selector_top_kPK11llama_model",
    ],
    [llama_model_p_ctypes],
    ctypes.c_int32,
    required=False,
)
def llama_model_dflash_selector_top_k(
    model: llama_model_p
) -> int:
    """Return the DFlash2 selector width, or zero for DFlash v1/DSpark."""
    ...

# // returns pointer to the target-model layer indices
# LLAMA_API const int32_t * llama_model_target_layer_ids  (const struct llama_model * model);
@ctypes_function_llama_ext(
    [
        "llama_model_target_layer_ids",
        "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z",
        "__Z28llama_model_target_layer_idsPK11llama_model",
        "_Z28llama_model_target_layer_idsPK11llama_model",
    ],
    [llama_model_p_ctypes],
    ctypes.POINTER(ctypes.c_int32),
    required=False,
)
def llama_model_target_layer_ids(
    model: llama_model_p
) -> ctypes.POINTER(ctypes.c_int32):  # type: ignore
    """Return the target layer indices required by this draft model.

    Read llama_model_target_layer_ids_n(model) int32 entries. The returned
    array belongs to the model and must not be freed or modified. An empty
    array returns NULL; copy it before freeing the model.
    """
    ...

# // returns the number of extracted layers from target model
# LLAMA_API uint32_t        llama_model_target_layer_ids_n(const struct llama_model * model);
@ctypes_function_llama_ext(
    [
        "llama_model_target_layer_ids_n",
        "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z",
        "__Z30llama_model_target_layer_ids_nPK11llama_model",
        "_Z30llama_model_target_layer_ids_nPK11llama_model",
    ],
    [llama_model_p_ctypes],
    ctypes.c_uint32,
    required=False,
)
def llama_model_target_layer_ids_n(
    model: llama_model_p
) -> int:
    """Return the number of entries in llama_model_target_layer_ids(model).

    Zero means the model does not specify target layer indices.
    """
    ...

# // retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab)
# // returns total number of elements or 0 on error
# // if out is nullptr, returns the number of tokens without writing to out
# // caller must allocate enough memory for out before calling
# LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out);
@ctypes_function_llama_ext(
    [
        "llama_model_get_tok_embd",
        "?llama_model_get_tok_embd@@YAIPEBUllama_model@@PEAM@Z",
        "__Z24llama_model_get_tok_embdPK11llama_modelPf",
        "_Z24llama_model_get_tok_embdPK11llama_modelPf",
    ],
    [llama_model_p_ctypes, ctypes.POINTER(ctypes.c_float)],
    ctypes.c_uint32,
    required=False,
)
def llama_model_get_tok_embd(
    model: llama_model_p,
    out: Optional[ctypes.POINTER(ctypes.c_float)],  # type: ignore
) -> int:
    """Copy the model's token embedding matrix into caller-owned F32 storage.

    Pass out=None to query the required number of floats, then allocate at
    least that many c_float entries and call again to fill them. The matrix
    contains n_embd * n_vocab elements, not merely n_vocab elements.

    Returns:
        The number of float elements, or zero when the matrix is unavailable
        or cannot be converted. The caller retains ownership of out.
    """
    ...


# LLAMA_API uint32_t llama_model_get_tok_embd_rows(const struct llama_model * model,
#         const llama_token * tokens, uint32_t n_tokens, float * out);
@ctypes_function_llama_ext(
    [
        "llama_model_get_tok_embd_rows",
        "?llama_model_get_tok_embd_rows@@YAIPEBUllama_model@@PEBHIPEAM@Z",
        "__Z29llama_model_get_tok_embd_rowsPK11llama_modelPKijPf",
        "_Z29llama_model_get_tok_embd_rowsPK11llama_modelPKijPf",
    ],
    [llama_model_p_ctypes, llama_token_p, ctypes.c_uint32, ctypes.POINTER(ctypes.c_float)],
    ctypes.c_uint32,
    required=False,
)
def llama_model_get_tok_embd_rows(
    model: llama_model_p,
    tokens: CtypesArray[llama_token],
    n_tokens: int,
    out: CtypesArray[ctypes.c_float],
) -> int:
    """Copy token embedding rows into caller-owned F32 storage, in token order.

    Allocate n_tokens * llama_model_n_embd(model) floats for out. These are
    token rows, without multimodal deepstack features. Return the number of
    floats written, or 0 on error; NULL out is not a size query.
    """
    ...


# LLAMA_API uint32_t llama_model_n_tokens_delay(const struct llama_model * model);
@ctypes_function_llama_ext(
    [
        "llama_model_n_tokens_delay",
        "?llama_model_n_tokens_delay@@YAIPEBUllama_model@@@Z",
        "__Z26llama_model_n_tokens_delayPK11llama_model",
        "_Z26llama_model_n_tokens_delayPK11llama_model",
    ],
    [llama_model_p_ctypes],
    ctypes.c_uint32,
    required=False,
)
def llama_model_n_tokens_delay(model: llama_model_p) -> int:
    """Return realtime audio output delay in tokens, or 0 when not applicable."""
    ...