Notes

Write a note in this area. It's really easy to share with others. Click here ...import pdfplumber
from transformers import BertTokenizer, BertForTokenClassification
import torch

# Load BERT tokenizer and pre-trained NER model
tokenizer = BertTokenizer.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
model = BertForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")

# Load PDF and extract text
pdf_path = "path/to/your/pdf.pdf"
with pdfplumber.open(pdf_path) as pdf:
pdf_text = ""
for page in pdf.pages:
pdf_text += page.extract_text()

# Split text into smaller chunks that fit the model's input length
chunk_size = 200 # Define the chunk size
chunks = [pdf_text[i:i+chunk_size] for i in range(0, len(pdf_text), chunk_size)]

# Process each chunk
for chunk in chunks:
# Tokenize text and convert to model input format
tokens = tokenizer.tokenize(tokenizer.decode(tokenizer.encode(chunk)))
inputs = tokenizer.encode(chunk, return_tensors="pt")

# Perform NER using BERT model
with torch.no_grad():
outputs = model(inputs).logits

# Get predicted labels for each token
predicted_labels = torch.argmax(outputs, dim=2)[0]

# Map predicted labels back to tokens
predicted_tokens = tokenizer.convert_ids_to_tokens(inputs[0])
predicted_entities = [(token, model.config.id2label[label_id.item()]) for token, label_id in zip(predicted_tokens, predicted_labels)]

# Print extracted entities
for token, entity in predicted_entities:
if entity != "O": # "O" indicates no entity
print(f"Token: {token}, Entity: {entity}")