npx -y create-pinecone-app@latest --template legal-semantic-search 

PINECONE_API_KEY="{{YOUR_API_KEY}}"

# This API key has ReadWrite access to all indexes in your project.

cd legal-semantic-search && npm install 

# You must first activate a Billing Account here: https://www.voyageai.com/ 
# Then get your Voyage API Key here: https://dash.voyageai.com/
VOYAGE_API_KEY="your-api-key-here"

# Get your Pinecone API key here: https://app.pinecone.io/
PINECONE_API_KEY="your-api-key-here"
PINECONE_INDEX="legal_semantic_search" # Or any other name you wish

npm run dev

// For the purposes of our legal semantic search example, we pre-define some queries 
// that we know will pull back interesting results for the user
const suggestedSearches = [
  'Cases about personal freedoms being violated',
  'Cases involving a US President',
  'Cases involving guns',
  'Cases where Nixon was the defendant',
  'How much power does the commerce clause give Congress?',
  'Cases about personal rights or congressional overreach?',
  'Cases involving the ability to pay for an attorney',
  ...
];

// Then, we pass them into our SearchForm component: 
<SearchForm
  suggestedSearches={suggestedSearches}
  onSearch={(query: string) => {
    handleSearch(query, setResults, setIsSearching);
    setQuery(query);
  }}
/>

export const handleBootstrapping = async (targetIndex: string) => {

  try {
    console.log(`Running bootstrapping procedure against Pinecone index: ${targetIndex}`);

    // If a Pinecone index with the target name doesn't exist, create it
    // If it does exist, return while suppressing conflict errors
    await createIndexIfNecessary(targetIndex);

    // Short-circuit early if the index already exists and has vectors in it 
    const hasVectors = await pineconeIndexHasVectors(targetIndex);
    if (hasVectors) {
      console.log('Pinecone index already exists and has vectors in it - returning early without bootstrapping');
      return NextResponse.json({ sucess: true }, { status: 200 });
    }

    if (!hasVectors) {
      console.log('Pinecone index does not exist or has no vectors in it - bootstrapping');
    }

    // Load metadata from db.json
    const metadata = await readMetadata();

    // Form the local path to the PDFs documents
    const docsPath = path.resolve(process.cwd(), 'docs/')

    const loader = new DirectoryLoader(docsPath, {
      '.pdf': (filePath: string) => new PDFLoader(filePath),
    });

    // Load all PDFs within the specified directory
    const documents = await loader.load();

    // Merge extracted metadata with documents based on filename
    documents.forEach((doc, index) => {
      const fileMetadata = metadata.find(meta => meta.filename === path.basename(doc.metadata.source));
      if (fileMetadata) {
        doc.metadata = { ...doc.metadata, ...fileMetadata, pageContent: doc.pageContent };
      } else {
        console.warn(`No metadata found for ${doc.metadata.source}`);
      }
    });

    // Split text into chunks
    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
    const splits = await splitter.splitDocuments(documents);

    // Assign unique IDs to each split and flatten metadata
    const castedSplits: Document[] = splits.map(split => ({
      pageContent: split.pageContent,
      metadata: {
        ...flattenMetadata(split.metadata as Document['metadata']),
        id: uuidv4(),
        pageContent: split.pageContent, // Ensure pageContent is included in metadata
      },
    }));

    // Extract page contents
    const pageContents = castedSplits.map(split => split.pageContent);

    // Generate embeddings for each chunk
    const voyageEmbeddings = new VoyageEmbeddings({
      apiKey: process.env.VOYAGE_API_KEY,
      inputType: "document",
      modelName: "voyage-law-2",
    });

    const embeddings = await voyageEmbeddings.embedDocuments(pageContents);

    // Combine embeddings with metadata and ensure IDs are defined
    const vectors = castedSplits.map((split, index) => {
      if (!split.metadata.id) {
        throw new Error('Document chunk is missing an ID');
      }
      return {
        id: split.metadata.id!,
        values: embeddings[index],
        metadata: split.metadata,
      };
    });

    const pc = new Pinecone();
    const index = pc.Index(process.env.PINECONE_INDEX as string);

    // Batch upserts to stay within the 2MB request size limit
    await batchUpserts(index, vectors);

    console.log('Bootstrap procedure completed.');
    return NextResponse.json({ success: true }, { status: 200 });

  } catch (error) {
    console.error('Error during bootstrap procedure:', error);
  }

  return true;
}

// Generate embeddings for each chunk
const voyageEmbeddings = new VoyageEmbeddings({
  apiKey: process.env.VOYAGE_API_KEY,
  inputType: "document",
  modelName: "voyage-law-2",
});

// Initialize VoyageEmbeddings
const voyageEmbeddings = new VoyageEmbeddings({
  apiKey: process.env.VOYAGE_API_KEY,
  inputType: 'document',
  modelName: "voyage-law-2",
});

// Initialize PineconeVectorStore
const vectorStore = new PineconeStore(voyageEmbeddings, {
  pineconeIndex: pc.Index(process.env.PINECONE_INDEX as string),
});

const retrieved = await vectorStore.maxMarginalRelevanceSearch(query, { k: 20 });