Spaces:
Running
Running
File size: 2,222 Bytes
9c28499 588ba9b 9c28499 588ba9b 9c28499 57dc0e4 96ef181 57dc0e4 588ba9b 9c28499 0eae068 aae1ac3 9c28499 0eae068 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
"""
Configuration file for LLM provider
Change LLM_PROVIDER to switch between different models
"""
import os
# Swappable LLM provider (environment configurable)
LLM_PROVIDER = os.getenv("LLM_PROVIDER", "beam") # Options: "beam", "huggingface", "local"
# API Keys (set these as environment variables in HuggingFace Space secrets)
BEAM_API_URL = os.getenv("BEAM_API_URL", "")
BEAM_API_TOKEN = os.getenv("BEAM_API_TOKEN", "")
HUGGINGFACE_API_KEY = os.getenv("HUGGINGFACE_API_KEY", "")
# Model configurations
HUGGINGFACE_MODEL = "google/gemma-2-2b-it"
# Local model configuration (for quantized models hosted within the Space)
LOCAL_MODEL_REPO = os.getenv("LOCAL_MODEL_REPO", "bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF")
LOCAL_MODEL_FILENAME = os.getenv("LOCAL_MODEL_FILENAME", "Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf") # Q4_K_M (2.50GB, recommended)
LOCAL_MODEL_CONTEXT_LENGTH = int(os.getenv("LOCAL_MODEL_CONTEXT_LENGTH", "2048"))
LOCAL_MODEL_THREADS = int(os.getenv("LOCAL_MODEL_THREADS", str(os.cpu_count() or 2))) # HF Spaces has 2 vCPUs
LOCAL_MODEL_BATCH_SIZE = int(os.getenv("LOCAL_MODEL_BATCH_SIZE", "1024")) # Optimal for CPU throughput
LOCAL_MODEL_MAX_OUTPUT_TOKENS = int(os.getenv("LOCAL_MODEL_MAX_OUTPUT_TOKENS", "100")) # Shorter responses for faster UX
LOCAL_MODEL_HF_TOKEN = os.getenv("LOCAL_MODEL_HF_TOKEN", HUGGINGFACE_API_KEY or "")
# Access control configuration
CLIENT_APP_ORIGINS = [
origin.strip()
for origin in os.getenv("CLIENT_APP_ORIGINS", "").split(",")
if origin.strip()
]
API_ACCESS_TOKEN = os.getenv("API_ACCESS_TOKEN", "")
# Session token configuration
SESSION_TOKEN_SECRET = os.getenv("SESSION_TOKEN_SECRET", "")
SESSION_TOKEN_TTL_SECONDS = int(os.getenv("SESSION_TOKEN_TTL_SECONDS", "600"))
# RAG Configuration
EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2" # Fast, lightweight
CHUNK_SIZE = 300 # Characters per chunk (reduced for faster inference)
CHUNK_OVERLAP = 30 # Overlap between chunks
TOP_K_RESULTS = 3 # Retrieve top 3 most relevant chunks (more context for GPU inference)
# System prompt for the chatbot
SYSTEM_PROMPT = """Answer questions about Bi using the provided context. Keep answers short and direct. Always refer to Bi by name."""
|