Notes

!pip install -q transformers einops accelerate langchain bitsandbytes

!nvidia-smi

from langchain import HuggingFacePipeline
from transformers import AutoTokenizer, pipeline
import torch

model = "tiiuae/falcon-7b-instruct" #tiiuae/falcon-40b-instruct

tokenizer = AutoTokenizer.from_pretrained(model)

pipeline = pipeline(
"text-generation", #task
model=model,
tokenizer=tokenizer,
torch_dtype=torch.bfloat16,
trust_remote_code=True,
device_map="auto",
max_length=200,
do_sample=True,
top_k=10,
num_return_sequences=1,
eos_token_id=tokenizer.eos_token_id
)

llm = HuggingFacePipeline(pipeline = pipeline, model_kwargs = {'temperature':0})

from langchain import PromptTemplate, LLMChain

template = """
You are an intelligent chatbot. Help me to summarize mail from question.
Question: {question}
Answer:"""
prompt = PromptTemplate(template=template, input_variables=["question"])

llm_chain = LLMChain(prompt=prompt, llm=llm)

question = """summarize the mail:

Hi,
Including one of my Admins who works with our internal IT team on those suggested solutions.
LUKASZ DYCHALA
Technical Team Lead

IG, ul. Kapelanka 42B, Kraków, 30-347
D: +48123762041 | T: +442078960011
www.ig.com

45 YEARS OF TRADING
INDICES | SHARES | FOREX
COMMODITIES"""

print(llm_chain.run(question))
add the prompt template same as above in below codes
!pip install accelerate==0.20.3
!pip install peft
!pip install appdirs==1.4.4
!pip install bitsandbytes==0.41.1
!pip install datasets==2.10.1
!pip install fire==0.5.0
#pip uninstall peft -y
#!pip install git+https://github.com/huggingface/peft.git@e536616888d51b453ed354a6f1e243fecb02ea08
!pip install git+https://github.com/huggingface/transformers.git
!pip install sentencepiece==0.1.97
!pip install tensorboardX==2.6
!pip install gradio==3.23.0

# Commented out IPython magic to ensure Python compatibility.
import transformers
import textwrap
from transformers import LlamaTokenizer, LlamaForCausalLM, AutoModel, AutoModelForCausalLM, LlamaForCausalLM
from peft import PeftModel
import os
import sys
from typing import List
from peft import (
LoraConfig,
get_peft_model,
get_peft_model_state_dict,
prepare_model_for_int8_training,
)

import fire
import torch
from datasets import load_dataset
import pandas as pd

import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns
from pylab import rcParams

# %matplotlib inline
sns.set(rc={'figure.figsize':(10, 7)})
sns.set(rc={'figure.dpi':100})
sns.set(style='white', palette='muted', font_scale=1.2)

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
DEVICE

from huggingface_hub import notebook_login

notebook_login()

BASE_MODEL = "meta-llama/Llama-2-7b-chat-hf"
#adapters_name = '/content/llama_fine_tuned'


model = LlamaForCausalLM.from_pretrained(
BASE_MODEL,
load_in_8bit=True,
torch_dtype=torch.float16,
quantization_config=None,
use_auth_token='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'
)


tokenizer = LlamaTokenizer.from_pretrained(BASE_MODEL)

tokenizer.pad_token_id = (
0 # unk. we want this to be different from the eos token
)
tokenizer.padding_side = "left"

def response_text(input, m, tok):
input_ids = tok.encode(input, return_tensors="pt").to(m.device)
output = m.generate(input_ids, max_new_tokens=200, pad_token_id=tok.eos_token_id)

# Decode the reponsegenerated response
response = tok.decode(output[0], skip_special_tokens=True)

# Find the last complete sentence in the response
sentences = response.rsplit(".", 1)[0] + "."
sentences = sentences.split('n')
sentences = sentences[1:]

# Remove the first sentence if it matches the input question exactly
if sentences[0].strip().lower().rstrip('?') == input.strip().lower():
sentences = sentences[1:]

# Remove the last sentence if it starts with a number followed by a period
if sentences[-1].strip().startswith(tuple("0123456789")) and sentences[-1].strip().endswith("."):
sentences = sentences[:-1]

# Remove any duplicate sentences
unique_sentences = []
for sentence in sentences:
if sentence.strip() not in unique_sentences:
unique_sentences.append(sentence.strip())

new_text = 'n'.join(unique_sentences)
return new_text

input_text = "generate an email regarding update of sentiment analysis"

try:
response = response_text(input_text, model, tokenizer)
print(response)
except Exception as e:
print("An error occurred:", str(e))