voyage-multimodal-3

!pip install -qU voyageai pinecone

from pinecone import Pinecone, ServerlessSpec

pc = Pinecone(api_key="API_KEY")

# Create Index
index_name = "voyage-multimodal-3"

if not pc.has_index(index_name):
    pc.create_index(
        name=index_name,
        dimension=1024,
        metric="cosine",
        spec=ServerlessSpec(
            cloud="aws",
            region="us-east-1"
        )
    )

index = pc.Index(index_name)

from typing import Union

# Embed data
data = [
    {"id": "vec1", "data": {"content": [{"type": "image_url", "image_url": "https://example.com/image.jpg"}, {"type": "text", "text": "Frontier intelligence at 2x the speed"}]}},
    {"id": "vec2", "data": {"content": [{"type": "image_url", "image_url": "https://example.com/page1.jpg"}, {"type": "image_url", "image_url": "https://example.com/page2.jpg"}]}},
    {"id": "vec3", "data": {"content": [{"type": "image_base64", "image_base64": "data:image/jpeg;base64,..."}]}}
]

# You can also use lists of texts and PIL Images, e.g.:
# "data": ["This is a banana", PIL.Image.open("banana.jpg")] ]

import voyageai

vo = voyageai.Client(api_key=VOYAGE_API_KEY)

model_id = "voyage-multimodal-3"

def embed(docs: Union[list[dict[str, str]], list[list[Union[str, Image]]]], input_type: str) -> list[list[float]]:
    embeddings = vo.multimodal_embed(
		    docs,
		    model=model_id,
		    input_type=input_type
		).embeddings
    return embeddings

# Use "document" input type for documents
embeddings = embed([d["data"] for d in data], input_type="document")

vectors = []
for d, e in zip(data, embeddings):
    vectors.append({
        "id": d['id'],
        "values": e,
        "metadata": {'inputs': d['inputs']}
    })

index.upsert(
    vectors=vectors,
    namespace="ns1"
)

query = ["Strong LLMs in 2024"]

# Use "query" input type for queries
x = embed([query], input_type="query")

results = index.query(
    namespace="ns1",
    vector=x[0],
    top_k=3,
    include_values=False,
    include_metadata=True
)

print(results)