Offline AI Agent: AI That Works Without Internet

Offline AI Agent: AI That Works Without Internet

💡 What You Will Learn

Offline AI Agent: AI That Works Without Internet

|:----|:--------|:--------|:--------| || Ollama | llama.cpp, vLLM || || ChromaDB | FAISS, SQLite-vss || || BGE-small-zh-v1.5 | text2vec-base-chinese || || Qwen3-7B-Q4_K_M | DeepSeek-R1-Distill-Qwen-7B || || — | — ||

Step 1InstallationOllama

# Installation/Setup
# 
curl -fsSL https://ollama.com/install.sh | sh

# 
ollama pull qwen3:7b-q4_K_M
ollama pull nomic-embed-text

# Installation/Setup
ollama run qwen3:7b-q4_K_M "Introduce yourself in one sentence"

import chromadb
from chromadb.utils import embedding_functions

class OfflineRAG:
    """RAG"""

    def __init__(self, persist_dir="./offline_rag_db"):
        self.emb_fn = embedding_functions.SentenceTransformerEmbeddingFunction(
            model_name="BAAI/bge-small-zh-v1.5",
            device="cpu"
        )
        self.client = chromadb.PersistentClient(path=persist_dir)
        self.collection = self.client.get_or_create_collection(
            name="knowledge_base",
            embedding_function=self.emb_fn
        )

    def add_documents(self, documents: list, ids: list, metadatas: list = None):
        self.collection.add(
            documents=documents,
            ids=ids,
            metadatas=metadatas or [{}] * len(documents)
        )
        print(f" {len(documents)} ")

    def query(self, question: str, k: int = 3) -> str:
        results = self.collection.query(
            query_texts=[question],
            n_results=k
        )
        context = "\n\n".join(results['documents'][0])
        prompt = f"\n\n{context}\n\n{question}"
        return self._call_local_llm(prompt)

    def _call_local_llm(self, prompt: str) -> str:
        import requests
        response = requests.post(
            "http://localhost:11434/api/generate",
            json={
                "model": "qwen3:7b-q4_K_M",
                "prompt": prompt,
                "stream": False,
                "options": {
                    "temperature": 0.3,
                    "num_predict": 512
                }
            }
        )
        return response.json()["response"]

|:------|:------|:------|:----| || Q8_0 | Q4_K_M || || 32768 | 8192 || || 512 | 256 || || 0.7 | 0.3 ||

# StartOllama
ollama serve &

# 
curl http://localhost:11434/api/tags

# 
python3 offline_rag_setup.py --data-dir ./docs --persist-dir ./chroma_db

# StartAPI
python3 offline_api_server.py --port 8080

|:----|:-------------|:------------------| || 92% | 85% |