Skip to main content

​
Overview

Rich multimodal embedding model that can vectorize interleaved text and content-rich images, such as screenshots of PDFs, slides, tables, figures, and more. See blog post for details. Visit the Voyage documentation for an overview of all Voyage embedding models and rerankers.Access to models is through the Voyage Python client. You must register for Voyage API keys to access.

​
Using the model

​
Installation

!pip install -qU voyageai pinecone

​
Create Index

from pinecone import Pinecone, ServerlessSpec

pc = Pinecone(api_key="API_KEY")

# Create Index
index_name = "voyage-multimodal-3"

if not pc.has_index(index_name):
    pc.create_index(
        name=index_name,
        dimension=1024,
        metric="cosine",
        spec=ServerlessSpec(
            cloud="aws",
            region="us-east-1"
        )
    )

index = pc.Index(index_name)

​
Embed & Upsert

from typing import Union

# Embed data
data = [
    {"id": "vec1", "data": {"content": [{"type": "image_url", "image_url": "https://example.com/image.jpg"}, {"type": "text", "text": "Frontier intelligence at 2x the speed"}]}},
    {"id": "vec2", "data": {"content": [{"type": "image_url", "image_url": "https://example.com/page1.jpg"}, {"type": "image_url", "image_url": "https://example.com/page2.jpg"}]}},
    {"id": "vec3", "data": {"content": [{"type": "image_base64", "image_base64": "data:image/jpeg;base64,..."}]}}
]

# You can also use lists of texts and PIL Images, e.g.:
# "data": ["This is a banana", PIL.Image.open("banana.jpg")] ]

import voyageai

vo = voyageai.Client(api_key=VOYAGE_API_KEY)

model_id = "voyage-multimodal-3"

def embed(docs: Union[list[dict[str, str]], list[list[Union[str, Image]]]], input_type: str) -> list[list[float]]:
    embeddings = vo.multimodal_embed(
		    docs,
		    model=model_id,
		    input_type=input_type
		).embeddings
    return embeddings

# Use "document" input type for documents
embeddings = embed([d["data"] for d in data], input_type="document")

vectors = []
for d, e in zip(data, embeddings):
    vectors.append({
        "id": d['id'],
        "values": e,
        "metadata": {'inputs': d['inputs']}
    })

index.upsert(
    vectors=vectors,
    namespace="ns1"
)

​
Query

query = ["Strong LLMs in 2024"]

# Use "query" input type for queries
x = embed([query], input_type="query")

results = index.query(
    namespace="ns1",
    vector=x[0],
    top_k=3,
    include_values=False,
    include_metadata=True
)

print(results)
Lorem Ipsum