panda_guard.llms package

Submodules

panda_guard.llms.base module

class panda_guard.llms.base.BaseLLM(config: BaseLLMConfig)[source]

Bases: ABC

Abstract base class for LLM.

Parameters:: config – Configuration object for LLM.

property avg_tokens

Get the average number of tokens per request.

Returns:: Average number of tokens.

batch_generate(batch_messages: List[List[Dict[str, str]]], config: LLMGenerateConfig) → List[List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]]][source]

Generate responses for a batch of messages concurrently.

Parameters:

batch_messages – List of batches of messages.
config – Configuration for generation.

Returns:

List of generated responses.

abstract continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]][source]

Remove EOS token in formatted prompt. Manually add generation prompt.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response or responses with log probabilities.

abstract evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig, require_grad=False) → List[float][source]

Abstract method for evaluating log likelihood of messages.

Parameters:

messages – List of messages to evaluate.
config – Configuration for generation.
require_grad – Whether grad information is needed.

Returns:

List of log likelihoods.

abstract generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]][source]

Abstract method for generating response from LLM.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response or responses with log probabilities.

reset()[source]: Reset the token counts and the number of requests.

property total_tokens

Get the total number of tokens used.

Returns:: Total number of tokens.

update(prompt_tokens: int, completion_tokens: int, num_requires: int)[source]

Update the token counts and number of requests.

Parameters:

prompt_tokens – Number of tokens in prompt.
completion_tokens – Number of tokens in completion.
num_requires – Number of requests made.

class panda_guard.llms.base.BaseLLMConfig(llm_type: str | None = None, model_name: str | None = None)[source]

Bases: ABC

Base configuration for LLM.

Parameters:

llm_type – Type of the LLM.
model_name – Name of the model.

llm_type: str = None

model_name: str = None

class panda_guard.llms.base.LLMGenerateConfig(max_n_tokens: int | None = None, temperature: float | None = None, logprobs: bool = False, seed: int | None = None, stream: bool = False)[source]

Bases: object

Configuration for LLM generation.

Parameters:

max_n_tokens – Maximum number of tokens to generate.
temperature – Temperature for sampling randomness.
logprobs – Whether to return log probabilities.
seed – Seed for reproducibility.
stream – Whether to use streaming generation.

logprobs: bool = False

max_n_tokens: int = None

seed: int = None

stream: bool = False

temperature: float = None

panda_guard.llms.claude module

class panda_guard.llms.claude.ClaudeLLM(config: ClaudeLLMConfig)[source]

Bases: BaseLLM

Claude LLM Implementation.

Parameters:: config – Configuration for Claude LLM.

continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig)[source]

Generate continuation for the last message.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response.

evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig, require_grad=False) → List[float][source]

Evaluate the log likelihood of the given messages.

Parameters:

messages – List of messages for evaluation.
config – Configuration for LLM generation.
require_grad – Whether to compute gradients (not supported for API models).

Raises:

NotImplementedError – Claude API does not support log likelihood evaluation.

generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → list[dict[str, str]] | Generator[str, None, None] | None[source]

Generate a response for a given input using Anthropic Claude API.

Parameters:

messages – List of input messages.
config – Configuration for LLM generation.

Returns:

Generated response, stream generator, or response with logprobs.

class panda_guard.llms.claude.ClaudeLLMConfig(llm_type: str = 'ClaudeLLM', model_name: str = 'claude-3-opus-20240229', api_key: str | None = None, max_tokens_to_sample: int | None = None)[source]

Bases: BaseLLMConfig

Claude LLM Configuration.

Parameters:

llm_type – Type of LLM, default is “ClaudeLLM”.
model_name – Name of the model.
api_key – API key for accessing Anthropic.
max_tokens_to_sample – Maximum tokens to sample, overrides max_n_tokens if provided.

api_key: str = None

llm_type: str = 'ClaudeLLM'

max_tokens_to_sample: int = None

model_name: str = 'claude-3-opus-20240229'

panda_guard.llms.gemini module

class panda_guard.llms.gemini.GeminiLLM(config: GeminiLLMConfig)[source]

Bases: BaseLLM

Gemini LLM Implementation.

Parameters:: config – Configuration for Gemini LLM.

continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig)[source]

Generate continuation for the last message.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response.

evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig, require_grad=False) → List[float][source]

Evaluate the log likelihood of the given messages.

Parameters:

messages – List of messages for evaluation.
config – Configuration for LLM generation.
require_grad – Whether to compute gradients (not supported for API models).

Raises:

NotImplementedError – Gemini API does not support log likelihood evaluation.

generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]] | Generator[str, None, None][source]

Generate a response for a given input using Google Gemini API.

Parameters:

messages – List of input messages.
config – Configuration for LLM generation.

Returns:

Generated response, stream generator, or response with logprobs.

class panda_guard.llms.gemini.GeminiLLMConfig(llm_type: str = 'GeminiLLM', model_name: str = 'gemini-1.5-pro', api_key: str | None = None, safety_settings: Dict[str, str] | None = None)[source]

Bases: BaseLLMConfig

Gemini LLM Configuration.

Parameters:

llm_type – Type of LLM, default is “GeminiLLM”.
model_name – Name of the model.
api_key – API key for accessing Google AI.
safety_settings – Custom safety settings for the model.

api_key: str = None

llm_type: str = 'GeminiLLM'

model_name: str = 'gemini-1.5-pro'

safety_settings: Dict[str, str] = None

panda_guard.llms.hf module

class panda_guard.llms.hf.HuggingFaceLLM(config: HuggingFaceLLMConfig)[source]

Bases: BaseLLM

Hugging Face Language Model Implementation.

Parameters:: config – Configuration for Hugging Face LLM.

batch_generate(batch_messages: List[List[Dict[str, str]]], config: LLMGenerateConfig) → List[List[Dict[str, str]]][source]

Generate responses for a batch of messages in a single call.

Parameters:

batch_messages – List of batches of messages.
config – Configuration for LLM generation.

Returns:

List of generated responses for each batch.

continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig)[source]

Remove EOS token in formatted prompt. Manually add generation prompt.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response or responses with log probabilities.

evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig, require_grad=False) → List[float] | List[Tensor][source]

Evaluate the log likelihood of the given messages.

Parameters:

messages – List of messages for evaluation.
config – Configuration for LLM generation.
require_grad – Whether to compute gradients (not supported for API models).

Returns:

List of log likelihood values.

generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]] | Generator[str, None, None][source]

Generate a response for a given input using Hugging Face model.

Parameters:

messages – List of input messages.
config – Configuration for LLM generation.

Returns:

Generated response, stream generator, or response with logprobs.

class panda_guard.llms.hf.HuggingFaceLLMConfig(llm_type: str = 'HuggingFaceLLM', model_name: [<class 'str'>, typing.Any] = None, device_map: str = 'auto')[source]

Bases: BaseLLMConfig

Hugging Face LLM Configuration.

Parameters:

llm_type – Type of LLM, default is “HuggingFaceLLM”.
model_name – Name of the model or model instance.
device_map – Device mapping for model deployment.

device_map: str = 'auto'

llm_type: str = 'HuggingFaceLLM'

model_name: [<class 'str'>, typing.Any] = None

panda_guard.llms.oai module

class panda_guard.llms.oai.OpenAiChatLLM(config: OpenAiLLMConfig)[source]

Bases: BaseLLM

OpenAI Chat LLM Implementation.

Parameters:: config – Configuration for OpenAI Chat LLM.

continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig)[source]

Remove EOS token in formatted prompt. Manually add generation prompt.

Parameters:

messages – List of messages for input.
config – Configuration for LLM generation.

Raises:

NotImplementedError – OpenAiChatLLM does not support continual generation.

evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[float][source]

Evaluate the log likelihood of the given messages.

Parameters:

messages – List of messages for evaluation.
config – Configuration for LLM generation.

Raises:

NotImplementedError – OpenAI Chat does not support log likelihood evaluation.

generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → list[dict[str, str]] | tuple[list[dict[str, str]], list[Any]] | Generator[str, None, None] | None[source]

Generate a response for a given input using OpenAI Chat API.

Parameters:

messages – List of input messages.
config – Configuration for LLM generation.

Returns:

Generated response or response with logprobs or stream generator.

class panda_guard.llms.oai.OpenAiChatLLMConfig(llm_type: str = 'OpenAiChatLLM', model_name: str | None = None, base_url: str | None = None, api_key: str = 'KEY HERE')[source]

Bases: BaseLLMConfig

OpenAI Chat LLM Configuration.

Parameters:

llm_type – Type of LLM, default is “OpenAiChatLLM”.
model_name – Name of the model.
base_url – Base URL for the OpenAI API.
api_key – API key for accessing OpenAI.

api_key: str = 'KEY HERE'

base_url: str = None

llm_type: str = 'OpenAiChatLLM'

model_name: str = None

class panda_guard.llms.oai.OpenAiLLM(config: OpenAiLLMConfig)[source]

Bases: BaseLLM

OpenAI LLM Implementation.

Parameters:: config – Configuration for OpenAI LLM.

continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig)[source]

Remove EOS token in formatted prompt. Manually add generation prompt.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response or responses with log probabilities.

evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig, require_grad=False) → List[float][source]

Evaluate the log likelihood of the given messages.

Parameters:

messages – List of messages for evaluation.
config – Configuration for LLM generation.

Returns:

List of log likelihood values.

generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → list[dict[str, str]] | tuple[list[dict[str, str]], list[float] | None] | Generator[str, Any, None] | None[source]

Generate a response for a given input using OpenAI API.

Parameters:

messages – List of input messages.
config – Configuration for LLM generation.

Returns:

Generated response or response with logprobs or stream generator.

class panda_guard.llms.oai.OpenAiLLMConfig(llm_type: str = 'OpenAiLLM', model_name: str | None = None, base_url: str | None = None, api_key: str = 'KEY HERE')[source]

Bases: BaseLLMConfig

OpenAI LLM Configuration.

Parameters:

llm_type – Type of LLM, default is “OpenAiLLM”.
model_name – Name of the model.
base_url – Base URL for the OpenAI API.
api_key – API key for accessing OpenAI.

api_key: str = 'KEY HERE'

base_url: str = None

llm_type: str = 'OpenAiLLM'

model_name: str = None

panda_guard.llms.ollama_llm module

panda_guard.llms.sglang_llm module

panda_guard.llms.vllm_llm module

class panda_guard.llms.vllm_llm.VLLMLLM(config: VLLMLLMConfig)[source]

Bases: BaseLLM

VLLM LLM Implementation for high-performance inference.

Parameters:: config – Configuration for VLLM LLM.

batch_generate(batch_messages: List[List[Dict[str, str]]], config: LLMGenerateConfig) → List[List[Dict[str, str]]][source]

Generate responses for a batch of messages in one go using VLLM’s batching capabilities.

Parameters:

batch_messages – List of batches of messages.
config – Configuration for LLM generation.

Returns:

List of generated responses for each batch.

continual_generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]][source]

Generate continuation for the existing conversation.

Parameters:

messages – List of messages for input.
config – Configuration for generation.

Returns:

Generated response or responses with log probabilities.

evaluate_log_likelihood(messages: List[Dict[str, str]], config: LLMGenerateConfig, require_grad=False) → List[float][source]

Evaluate the log likelihood of the given messages.

Parameters:

messages – List of messages for evaluation.
config – Configuration for LLM generation.
require_grad – Whether grad information is needed (not supported in VLLM)

Returns:

List of log likelihood values.

generate(messages: List[Dict[str, str]], config: LLMGenerateConfig) → List[Dict[str, str]] | Tuple[List[Dict[str, str]], List[float]] | Generator[str, None, None][source]

Generate a response using VLLM.

Parameters:

messages – List of input messages.
config – Configuration for LLM generation.

Returns:

Generated response, stream generator, or response with logprobs.

class panda_guard.llms.vllm_llm.VLLMLLMConfig(llm_type: str = 'VLLM', model_name: str | None = None, tensor_parallel_size: int = 1, gpu_memory_utilization: float = 0.9, max_model_len: int | None = None, quantization: str | None = None, trust_remote_code: bool = True)[source]

Bases: BaseLLMConfig

VLLM LLM Configuration.

Parameters:

llm_type – Type of LLM, default is “VLLMLLM”.
model_name – Name or path of the model.
tensor_parallel_size – Number of GPUs to use for tensor parallelism.
gpu_memory_utilization – Fraction of GPU memory to use.
max_model_len – Maximum sequence length.
quantization – Quantization method to use.
trust_remote_code – Whether to trust remote code.

gpu_memory_utilization: float = 0.9

llm_type: str = 'VLLM'

max_model_len: int | None = None

model_name: str = None

quantization: str | None = None

tensor_parallel_size: int = 1

trust_remote_code: bool = True

Module contents

panda_guard.llms.create_llm(config: BaseLLMConfig) → BaseLLM[source]