Document Ingestion Quickstart

import json
import os
from typing import Optional

from pydantic import BaseModel, Field
from tensorlake.documentai import DocumentAI
from tensorlake.documentai.models import ParsingOptions, StructuredExtractionOptions
from tensorlake.documentai.models.enums import ChunkingStrategy

doc_ai = DocumentAI(api_key=os.getenv("TENSORLAKE_API_KEY"))

# Use a publicly accessible URL or upload a file to Tensorlake and use the file ID.
file_url = "https://pub-226479de18b2493f96b64c6674705dd8.r2.dev/real-estate-purchase-all-signed.pdf"

# Define a JSON schema using Pydantic
# Our structured extraction model will identify the properties we want to extract from the document.
# In this case, we are extracting the names and signature dates of the buyer and seller.
class Signers(BaseModel):
    buyer_name: Optional[str] = Field(
        default=None, description="The name of the buyer, do not extract initials"
    )
    buyer_signature_date: Optional[str] = Field(
        default=None, description="Date and time that the buyer signed."
    )
    seller_name: Optional[str] = Field(
        default=None, description="The name of the seller, do not extract initials"
    )
    seller_signature_date: Optional[str] = Field(
        default=None, description="Date and time that the seller signed."
    )

# Create a structured extraction options object with the schema
#
# You can send as many schemas as you want, and the API will return structured data for each schema
# indexed by the schema name.
real_estate_agreement_extraction_options = StructuredExtractionOptions(
    schema_name="Signers",
    json_schema=Signers,
)

# Tune in the options for finding data in the document
#
# We provide sane defaults, but every document is different, so you can adjust
# every option to your needs.
#
# In this example, we are using the PAGE chunking strategy, which means that each page of the document will be a separate chunk.
parsing_options = ParsingOptions(
    chunking_strategy=ChunkingStrategy.PAGE,
)

# Submit the parse operation and wait for the job to complete
parse_id = doc_ai.parse(
    file=file_url,
    page_range="9-10",
    parsing_options=parsing_options,
    structured_extraction_options=[real_estate_agreement_extraction_options],
)

result = doc_ai.wait_for_completion(parse_id)

markdown_chunks = result.chunks
with open("markdown_chunks.md", "w") as f:
    for chunk_number, chunk in enumerate(markdown_chunks):
        f.write(f"## CHUNK NUMBER {chunk_number}\n\n")
        f.write(f"## Page {chunk.page_number}\n\n{chunk.content}\n\n")

serializable_data = result.model_dump()
with open("structured_data.json", "w") as f:
    json.dump(serializable_data["structured_data"], f, indent=2)

  async function parseFileUrl(fileUrl, tensorlakeApiKey) {
    const signersSchema = {
      title: "Signers",
      type: "object",
      properties: {
        buyerName: {
          type: "string",
          description: "The name of the buyer, do not extract initials",
          title: "Buyer Name"
        },
        buyerSignatureDate: {
          type: "string",
          description: "Date and time that the buyer signed.",
          title: "Buyer Signature Date"
        },
        sellerName: {
          type: "string",
          description: "The name of the seller, do not extract initials",
          title: "Seller Name"
        },
        sellerSignatureDate: {
          type: "string",
          description: "Date and time that the seller signed.",
          title: "Seller Signature Date"
        }
      }
    };

    const realEstateAgreementExtractionOptions = {
      schema_name: "Signers",
      json_schema: signersSchema,
    };

    const parsingOptions = {
      chunking_strategy: "page",
    };

    const body = {
      file_url: fileUrl,
      page_range: "9-10",
      parsing_options: parsingOptions,
      structured_extraction_options: [realEstateAgreementExtractionOptions],
    };

    const options = {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${tensorlakeApiKey}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(body),
    };

    const response = await fetch(
      'https://api.tensorlake.ai/documents/v2/parse',
      options
    );

    const result = await response.json();
    console.log('result:', JSON.stringify(result, null, 2));
    return result.jobId;
  }

const fileId =
  'https://pub-226479de18b2493f96b64c6674705dd8.r2.dev/real-estate-purchase-all-signed.pdf';
const tensorlakeApiKey =
  'your-tensorlake-api-key-here';

const parseId = await parseFileUrl(fileId, tensorlakeApiKey);

import { writeFileSync } from 'fs';

function writeParseResults(jobResult) {
  const structuredData = jobResult.structured_data;
  const markdownChunks = jobResult.chunks;

  writeFileSync(
    'structured_data.json',
    JSON.stringify(structuredData, null, 2)
  );

  let markdownContent = '';
  markdownChunks.forEach((chunk, chunkNumber) => {
    markdownContent += `## CHUNK NUMBER ${chunkNumber}\n\n`;
    markdownContent += `## Page ${chunk.page_number}\n\n${chunk.content}\n\n`;
  });

  writeFileSync('markdown_chunks.md', markdownContent);
}

async function getParseResults(parseId, tensorlakeApiKey) {
  while (true) {
    const response = await fetch(
      `https://api.tensorlake.ai/documents/v2/parse/${parseId}`,
      {
        method: 'GET',
        headers: {
          Authorization: `Bearer ${tensorlakeApiKey}`,
          'Content-Type': 'application/json',
        },
      }
    );
    if (!response.ok) {
      console.error(`Error fetching job: ${response.statusText}`);
      return;
    }

    const result = await response.json();

    if (result.status === 'pending' || result.status === 'processing') {
      console.log('waiting 5s...');
      await new Promise((resolve) => setTimeout(resolve, 5000));
      console.log(`job status: ${result.status}`);
    } else {
      if (result.status === 'successful') {
        console.log(result);
        writeParseResults(result);
        return result;
      } else {
        console.error(`Job finished with status: ${result.status}`);
        return result;
      }
    }
  }
}

const parseId = 'your-parse-id-here';
const tensorlakeApiKey = 'your-tensorlake-api-key-here';

await getParseResults(parseId, tensorlakeApiKey);