import os
from unstructured_ingest.pipeline.pipeline import Pipeline
from unstructured_ingest.interfaces import ProcessorConfig
from unstructured_ingest.processes.connectors.redisdb import (
RedisAccessConfig,
RedisConnectionConfig,
RedisUploaderConfig
)
from unstructured_ingest.processes.connectors.local import (
LocalIndexerConfig,
LocalConnectionConfig,
LocalDownloaderConfig
)
from unstructured_ingest.processes.partitioner import PartitionerConfig
from unstructured_ingest.processes.chunker import ChunkerConfig
from unstructured_ingest.processes.embedder import EmbedderConfig
# Chunking and embedding are optional.
if __name__ == "__main__":
Pipeline.from_configs(
context=ProcessorConfig(),
indexer_config=LocalIndexerConfig(input_path=os.getenv("LOCAL_FILE_INPUT_DIR")),
downloader_config=LocalDownloaderConfig(),
source_connection_config=LocalConnectionConfig(),
partitioner_config=PartitionerConfig(
partition_by_api=True,
api_key=os.getenv("UNSTRUCTURED_API_KEY"),
partition_endpoint=os.getenv("UNSTRUCTURED_API_URL"),
additional_partition_args={
"split_pdf_page": True,
"split_pdf_allow_failed": True,
"split_pdf_concurrency_level": 15
}
),
chunker_config=ChunkerConfig(chunking_strategy="by_title"),
embedder_config=EmbedderConfig(embedding_provider="huggingface"),
# Use a Redis connection string.
# destination_connection_config=RedisConnectionConfig(
# access_config=RedisAccessConfig(
# uri = os.getenv("REDIS_URI")
# )
# ),
# Use Redis connection properties.
destination_connection_config=RedisConnectionConfig(
access_config=RedisAccessConfig(
password=os.getenv("REDIS_PASSWORD")
),
host=os.getenv("REDIS_HOST"),
database=int(os.getenv("REDIS_DATABASE")),
port=int(os.getenv("REDIS_PORT")),
username=os.getenv("REDIS_USERNAME"),
ssl=False
),
uploader_config=RedisUploaderConfig(batch_size=100)
).run()
