RAG Fundamentals

Retrieval-Augmented Generation: Grounding LLM responses in external knowledge.

RAG Pipeline

Query → Embed → Retrieve → Rerank → Context → Generate → Verify

Core Components

Indexing

Document Loading

from llama_index.core import SimpleDirectoryReader

documents = SimpleDirectoryReader(
    "./data",
    recursive=True,
    required_exts=[".pdf", ".md", ".txt", ".html"]
).load_data()

Chunking Strategies

from llama_index.core.node_parser import (
    SentenceSplitter,
    SemanticSplitterNodeParser,
    CodeSplitter,
    MarkdownNodeParser
)

# Fixed
parser = SentenceSplitter(chunk_size=512, chunk_overlap=50)

# Semantic (uses embeddings)
parser = SemanticSplitterNodeParser(
    embed_model=embed_model,
    breakpoint_percentile_threshold=95
)

# Code-aware
parser = CodeSplitter(language="python", chunk_lines=50)

# Markdown (header-aware)
parser = MarkdownNodeParser()

Embedding Models

Vector Stores

# Pinecone
from llama_index.vector_stores import PineconeVectorStore
vector_store = PineconeVectorStore(
    index_name="my-index",
    namespace="docs",
    environment="us-west1-gcp"
)

# Qdrant
from llama_index.vector_stores import QdrantVectorStore
vector_store = QdrantVectorStore(
    host="localhost",
    port=6333,
    collection_name="docs"
)

# pgvector
from llama_index.vector_stores import PGVectorStore
vector_store = PGVectorStore.from_params(
    database="vectors",
    host="localhost",
    port=5432,
    table_name="embeddings"
)

Retrieval

Dense Retrieval (Semantic)

from llama_index.core.retrievers import VectorIndexRetriever

retriever = VectorIndexRetriever(
    index=index,
    similarity_top_k=10,
    vector_store_query_mode="default"
)

Sparse Retrieval (Keyword)

from llama_index.core.retrievers import KeywordTableRetriever

retriever = KeywordTableRetriever(
    index=index,
    similarity_top_k=10
)

Hybrid Retrieval

from llama_index.core.retrievers import HybridRetriever

retriever = HybridRetriever(
    dense_retriever=dense_retriever,
    sparse_retriever=sparse_retriever,
    dense_weight=0.7,
    sparse_weight=0.3
)

Multi-Vector Retrieval

from llama_index.core.retrievers import MultiVectorRetriever

# Parent document retrieval
retriever = MultiVectorRetriever(
    vectorstore=vectorstore,
    docstore=docstore,
    id_key="doc_id",
    search_kwargs={"k": 6}
)

Advanced Retrieval Patterns

Auto-Merging Retrieval

from llama_index.core.retrievers import AutoMergingRetriever

retriever = AutoMergingRetriever(
    vector_retriever=vector_retriever,
    min_chunk_size=512,
    merge_threshold=0.5
)

Recursive Retrieval

from llama_index.core.retrievers import RecursiveRetriever

retriever = RecursiveRetriever(
    "vector",
    retriever_dict={
        "vector": vector_retriever,
        "summary": summary_retriever
    },
    query_engine=query_engine
)

Hypothetical Document Embeddings (HyDE)

from llama_index.core.retrievers import HyDERetriever

retriever = HyDERetriever(
    vector_retriever=vector_retriever,
    llm=llm,
    include_original=True
)

Reranking

Cross-Encoder Reranker

from llama_index.postprocessor import CohereRerank, SentenceTransformerRerank

# Cohere (API)
reranker = CohereRerank(
    model="rerank-english-v3.0",
    top_n=5
)

# Local (Sentence Transformers)
reranker = SentenceTransformerRerank(
    model="cross-encoder/ms-marco-MiniLM-L-6-v2",
    top_n=5
)

LLM-Based Reranker

from llama_index.postprocessor import LLMRerank

reranker = LLMRerank(
    llm=llm,
    choice_batch_size=5,
    top_n=5
)

Pipeline

from llama_index.core.query_engine import RetrieverQueryEngine
from llama_index.core.postprocessor import SimilarityPostprocessor

query_engine = RetrieverQueryEngine.from_args(
    retriever=hybrid_retriever,
    node_postprocessors=[
        SimilarityPostprocessor(similarity_cutoff=0.7),
        CohereRerank(top_n=5),
        LongContextReorder()  # Put relevant in middle
    ]
)

Generation

Prompt Templates

from llama_index.core import PromptTemplate

QA_PROMPT = PromptTemplate(
    "Context:\n{context_str}\n\n"
    "Query: {query_str}\n\n"
    "Answer concisely. Cite sources as [doc_id].\n"
    "If unsure, say 'I don't know.'"
)

Response Modes

query_engine = index.as_query_engine(
    response_mode="tree_summarize",
    similarity_top_k=10,
    text_qa_template=QA_PROMPT
)

Streaming

streaming_response = query_engine.query("What is RAG?")
for token in streaming_response.response_gen:
    print(token, end="", flush=True)

Citations

from llama_index.core.response_synthesizers import CitationQueryEngine

query_engine = CitationQueryEngine.from_args(
    retriever=retriever,
    citation_chunk_size=512,
    citation_chunk_overlap=50
)

response = query_engine.query("What is RAG?")
# response.source_nodes contains citations
for node in response.source_nodes:
    print(f"[{node.node_id}] {node.text[:100]}")

Evaluation

Metrics

Faithfulness (Groundedness)

from llama_index.evaluation import FaithfulnessEvaluator

evaluator = FaithfulnessEvaluator(llm=llm)
result = evaluator.evaluate(
    query="What is RAG?",
    response="RAG combines retrieval and generation...",
    contexts=["RAG stands for Retrieval-Augmented Generation..."]
)
print(result.score)  # 0-1
print(result.feedback)

Relevancy

from llama_index.evaluation import RelevancyEvaluator

evaluator = RelevancyEvaluator(llm=llm)
result = evaluator.evaluate(
    query="What is RAG?",
    response="RAG combines retrieval...",
    contexts=["RAG stands for..."]
)

Advanced RAG Patterns

Agentic RAG (Agent as Retriever)

from llama_index.agent import ReActAgent

agent = ReActAgent.from_tools(
    tools=[QueryEngineTool(query_engine)],
    llm=llm
)

# Agent decides when/what to retrieve
response = agent.query("Compare RAG approaches")

Multi-Hop RAG

# Iterative retrieval
for hop in range(3):
    results = retriever.retrieve(query)
    query = refine_query(query, results)

Graph RAG

from llama_index.core import KnowledgeGraphIndex
from llama_index.graph_stores import Neo4jGraphStore

graph_store = Neo4jGraphStore(
    username="neo4j",
    password="...",
    url="bolt://localhost:7687"
)

kg_index = KnowledgeGraphIndex.from_documents(
    documents,
    graph_store=graph_store,
    max_triplets_per_chunk=10
)

Production Checklist

• [ ] Chunking strategy validated for domain
• [ ] Embedding model benchmarked
• [ ] Vector store scaled for QPS
• [ ] Hybrid retrieval implemented
• [ ] Reranker improves precision
• [ ] Citations enabled
• [ ] Faithfulness > 0.8
• [ ] Latency < 2s (p95)
• [ ] Cost per query tracked
• [ ] Feedback loop for continuous improvement

Related Notes

• Advanced RAG — HyDE, Graph RAG, Agentic RAG
• Memory Systems — RAG as memory
• Context Window Management — Context optimization
• Knowledge Graphs — Graph RAG deep dive
• Multimodal Context — Multimodal RAG
• LlamaIndex Agents — LlamaIndex implementation
• LangGraph — Agentic RAG with LangGraph
• Agent Benchmarks — RAG benchmarks

Sources: LlamaIndex RAG docs, "Retrieval-Augmented Generation" survey (Lewis et al.), "HyDE" (Gao et al.), "ReRank" (Nogueira et al.), "Graph RAG" (Edge et al.), Pinecone/Weaviate/Qdrant docs, Cohere rerank, Voyage AI embeddings, RAG evaluation papers.