做RAG需要自己准备一个txt文档,新建一个docs文件夹,放进去。例如,这里放了一个./docs/问答手册.txt
# 从指定文件读取,输入为List
from llama_index.core import SimpleDirectoryReader,Document
documents = SimpleDirectoryReader(input_files=['./docs/问答手册.txt']).load_data()
方法一:Documents可以直接构建index
# 构建向量索引
from llama_index.core import VectorStoreIndex
index = VectorStoreIndex.from_documents(documents,embed_model=embedding)
# 想要看到进度条的话,加一个参数 show_progress=True
# index = VectorStoreIndex.from_documents(documents,embed_model=embedding,show_progress=True)
方法二:可以先构建节点,再构建索引,同时采用faiss作为向量存储库
# 构建节点
from llama_index.core.node_parser import SentenceSplitter
transformations = [SentenceSplitter(chunk_size = 512)]from llama_index.core.ingestion.pipeline import run_transformations
nodes = run_transformations(documents, transformations=transformations)
根据节点构建索引
# 构建索引
from llama_index.vector_stores.faiss import FaissVectorStore
import faiss
from llama_index.core import StorageContext, VectorStoreIndex# 从上一节得知,智谱embedding-2的维度是1024
dimensions = len(emb)
vector_store = FaissVectorStore(faiss_index=faiss.IndexFlatL2(dimensions))
storage_context = StorageContext.from_defaults(vector_store=vector_store)index = VectorStoreIndex(nodes = nodes,storage_context=storage_context,embed_model = embedding,
)
这样索引就算是建成了。我们可以把索引存储到硬盘,这样以后就不用重复构建,直接从硬盘读取。
# save index to disk
persist_dir = "./storage"
index.storage_context.persist(persist_dir)
如果之前有保存过索引到硬盘,可以直接读取。
# load index from disk
from llama_index.vector_stores.faiss import FaissVectorStore
import faiss
from llama_index.core import StorageContext, load_index_from_storage
vector_store = FaissVectorStore.from_persist_dir(persist_dir)
storage_context = StorageContext.from_defaults(vector_store=vector_store, persist_dir=persist_dir
)
index = load_index_from_storage(storage_context=storage_context,embed_model = embedding)
index可以直接做问答引擎。
query_engine = index.as_query_engine(llm=llm)
# 回答提问
response = query_engine.query("What are the applications of Agent AI systems ?")
response
方法三:我们也可以先构建索引器,再构建合成器,再组装成问答引擎。
# 构建检索器
from llama_index.core.retrievers import VectorIndexRetriever
# 想要自定义参数,可以构造参数字典
kwargs = {'similarity_top_k': 5, 'index': index, 'dimensions': dimensions} # 必要参数
retriever = VectorIndexRetriever(**kwargs)
# 构建合成器
from llama_index.core.response_synthesizers import get_response_synthesizer
response_synthesizer = get_response_synthesizer(llm=llm)
# 构建问答引擎
from llama_index.core.query_engine import RetrieverQueryEngine
engine = RetrieverQueryEngine(retriever=retriever,response_synthesizer=response_synthesizer)
# 提问
question = "请问商标注册需要提供哪些文件?"
answer = engine.query(question)
print(answer.response)
方法四:利用Qdrant向量库
先安装一下
%pip install qdrant-client
%pip install llama-index-vector-stores-qdrant
%pip install llama-index-readers-file
加载文档
import qdrant_client
from llama_index.core import SimpleDirectoryReader# load documents
documents = SimpleDirectoryReader(input_files=['./docs/问答手册.txt']
).load_data()print("Document ID:", documents[0].doc_id)
Document ID: 02572b3e-18f7-4b5e-b432-3e1ed9ba89b8
构建索引
# Create an index over the documents
from llama_index.core import VectorStoreIndex, StorageContext
from llama_index.vector_stores.qdrant import QdrantVectorStore# 连接Qdrant,并保存在本地的qdrant文件夹中
qclient = qdrant_client.QdrantClient(path="qdrant")
vector_store = QdrantVectorStore(client=qclient, collection_name="wenda")
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex.from_documents(documents, storage_context=storage_context,embed_model = embedding
)
构建检索器
# 构建检索器
from llama_index.core.retrievers import VectorIndexRetriever
# 想要自定义参数,可以构造参数字典
kwargs = {'similarity_top_k': 5, 'index': index, 'dimensions': dimensions} # 必要参数
retriever = VectorIndexRetriever(**kwargs)
构建合成器
# 构建合成器
from llama_index.core.response_synthesizers import get_response_synthesizer
response_synthesizer = get_response_synthesizer(llm=llm)
构建问答引擎
# 构建问答引擎
from llama_index.core.query_engine import RetrieverQueryEngine
engine = RetrieverQueryEngine(retriever=retriever,response_synthesizer=response_synthesizer,)
提问
# 提问
question = "What are the applications of Agent AI systems ?"
answer = engine.query(question)
print(answer.response)
from llama_index.core.schema import TextNodenodes = [TextNode(text="The Shawshank Redemption",metadata={"author": "Stephen King","theme": "Friendship","year": 1994,},),TextNode(text="The Godfather",metadata={"director": "Francis Ford Coppola","theme": "Mafia","year": 1972,},),TextNode(text="Inception",metadata={"director": "Christopher Nolan","theme": "Fiction","year": 2010,},),TextNode(text="To Kill a Mockingbird",metadata={"author": "Harper Lee","theme": "Mafia","year": 1960,},),TextNode(text="1984",metadata={"author": "George Orwell","theme": "Totalitarianism","year": 1949,},),TextNode(text="The Great Gatsby",metadata={"author": "F. Scott Fitzgerald","theme": "The American Dream","year": 1925,},),TextNode(text="Harry Potter and the Sorcerer's Stone",metadata={"author": "J.K. Rowling","theme": "Fiction","year": 1997,},),
]
根据上面的nodes,构建索引。
vector_store = QdrantVectorStore(client=qclient, collection_name="filter")
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex(nodes, storage_context=storage_context,embed_model = embedding
)
然后我们就可以构建metadata filter了。
from llama_index.core.vector_stores import (MetadataFilter,MetadataFilters,FilterOperator,
)filters = MetadataFilters(filters=[MetadataFilter(key="theme", operator=FilterOperator.EQ, value="Mafia"),]
)
把构建好的filter当作参数,构建retriever。进行检索,查看一下结果。
retriever = index.as_retriever(filters=filters, llm=llm)
retriever.retrieve("What is inception about?")
我们还可以用AND或者OR来组合多个filter。
from llama_index.core.vector_stores import FilterOperator, FilterConditionfilters = MetadataFilters(filters=[MetadataFilter(key="theme", value="Fiction"),MetadataFilter(key="year", value=1997, operator=FilterOperator.GT),],condition=FilterCondition.AND,
)retriever = index.as_retriever(filters=filters, llm=llm)
retriever.retrieve("Harry Potter?")
我们也可以直接把filter的字典作为参数,构建retriever。这样可以构建一个更复杂的filter。
retriever = index.as_retriever(vector_store_kwargs={"filter": {"theme": "Mafia"}},llm=llm
)
retriever.retrieve("What is inception about?")
除了llama-index提供的检索方式,我们还可以利用Qdrant自带的检索能力。就是Default Qdrant Filters
nodes = [TextNode(text="りんごとは",metadata={"author": "Tanaka", "fruit": "apple", "city": "Tokyo"},),TextNode(text="Was ist Apfel?",metadata={"author": "David", "fruit": "apple", "city": "Berlin"},),TextNode(text="Orange like the sun",metadata={"author": "Jane", "fruit": "orange", "city": "Hong Kong"},),TextNode(text="Grape is...",metadata={"author": "Jane", "fruit": "grape", "city": "Hong Kong"},),TextNode(text="T-dot > G-dot",metadata={"author": "George", "fruit": "grape", "city": "Toronto"},),TextNode(text="6ix Watermelons",metadata={"author": "George","fruit": "watermelon","city": "Toronto",},),
]
继续构建向量库。
vector_store = QdrantVectorStore(client=qclient, collection_name="default")
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex(nodes, storage_context=storage_context,embed_model = embedding
)
构建Qdrant自己的的filter。
from qdrant_client.http.models import Filter, FieldCondition, MatchValue
filters = Filter(should=[Filter(must=[FieldCondition(key="fruit",match=MatchValue(value="apple"),),FieldCondition(key="city",match=MatchValue(value="Tokyo"),),]),Filter(must=[FieldCondition(key="fruit",match=MatchValue(value="grape"),),FieldCondition(key="city",match=MatchValue(value="Toronto"),),]),]
)
构建retriever。
retriever = index.as_retriever(vector_store_kwargs={"qdrant_filters": filters},llm=llm
)
检索一下看看
response = retriever.retrieve("Who makes grapes?")
for node in response:print("node", node.score)print("node", node.text)print("node", node.metadata)
本节内容参考了Llama-index官方文档