import os
from unstructured_ingest.pipeline.pipeline import Pipeline
from unstructured_ingest.interfaces import ProcessorConfig
from unstructured_ingest.processes.connectors.local import (
LocalIndexerConfig,
LocalDownloaderConfig,
LocalConnectionConfig
)
from unstructured_ingest.processes.partitioner import PartitionerConfig
from unstructured_ingest.processes.chunker import ChunkerConfig
from unstructured_ingest.processes.embedder import EmbedderConfig
# For Qdrant local:
# from unstructured_ingest.processes.connectors.qdrant.local import (
# LocalQdrantConnectionConfig,
# LocalQdrantAccessConfig,
# LocalQdrantUploadStagerConfig,
# LocalQdrantUploaderConfig
# )
# For Qdrant client-server:
# from unstructured_ingest.processes.connectors.qdrant.server import (
# ServerQdrantConnectionConfig,
# ServerQdrantAccessConfig,
# ServerQdrantUploadStagerConfig,
# ServerQdrantUploaderConfig
# )
# For Qdrant Cloud:
from unstructured_ingest.processes.connectors.qdrant.cloud import (
CloudQdrantConnectionConfig,
CloudQdrantAccessConfig,
CloudQdrantUploadStagerConfig,
CloudQdrantUploaderConfig
)
# Chunking and embedding are optional.
if __name__ == "__main__":
Pipeline.from_configs(
context=ProcessorConfig(),
indexer_config=LocalIndexerConfig(input_path=os.getenv("LOCAL_FILE_INPUT_DIR")),
downloader_config=LocalDownloaderConfig(),
source_connection_config=LocalConnectionConfig(),
partitioner_config=PartitionerConfig(
partition_by_api=True,
api_key=os.getenv("UNSTRUCTURED_API_KEY"),
partition_endpoint=os.getenv("UNSTRUCTURED_API_URL"),
additional_partition_args={
"split_pdf_page": True,
"split_pdf_allow_failed": True,
"split_pdf_concurrency_level": 15
}
),
chunker_config=ChunkerConfig(chunking_strategy="by_title"),
embedder_config=EmbedderConfig(embedding_provider="huggingface"),
# For Qdrant local:
# destination_connection_config=LocalQdrantConnectionConfig(
# access_config=LocalQdrantAccessConfig(),
# path=os.getenv("QDRANT_PATH")
# ),
# stager_config=LocalQdrantUploadStagerConfig(),
# uploader_config=LocalQdrantUploaderConfig(
# collection_name=os.getenv("QDRANT_COLLECTION"),
# batch_size=50,
# num_processes=1
# )
# For Qdrant client-server:
# destination_connection_config=ServerQdrantConnectionConfig(
# access_config=ServerQdrantAccessConfig(),
# url=os.getenv("QDRANT_URL")
# ),
# stager_config=ServerQdrantUploadStagerConfig(),
# uploader_config=ServerQdrantUploaderConfig(
# collection_name=os.getenv("QDRANT_COLLECTION"),
# batch_size=50,
# num_processes=1
# )
# For Qdrant cloud:
destination_connection_config=CloudQdrantConnectionConfig(
access_config=CloudQdrantAccessConfig(
api_key=os.getenv("QDRANT_API_KEY")
),
url=os.getenv("QDRANT_URL")
),
stager_config=CloudQdrantUploadStagerConfig(),
uploader_config=CloudQdrantUploaderConfig(
collection_name=os.getenv("QDRANT_COLLECTION"),
batch_size=50,
num_processes=1
)
).run()
