Notes

import pandas as pd
import numpy as np
from google.colab import drive
drive.mount('/content/drive')
#import data in the form of csv file
trans_data = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/transitions.csv',header=None)
#rd_data = pd.read_csv('rewards.csv',header=None)
rwd_data = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/rewards.csv',header=None)
#convert the pandas file into numpy array or matrix
trans_data = trans_data.to_numpy()
#t_data = pd.read_csv('transitions_data.csv',header=None)
rwd_data = rwd_data.to_numpy()
#preparation of the input data and storing it into a np matrix
transitions = {}
len_data = trans_data.shape[0]
td = trans_data
for i in range(1,len_data):
if (td[i][0] in transitions):
if td[i][1] in transitions[td[i][0]]:
transitions[td[i][0]][td[i][1]].append((float(td[i][3]),td[i][2]))
else:
transitions[td[i][0]][td[i][1]] = [(float(td[i][3]),td[i][2])]
else:
transitions[td[i][0]] = {td[i][1]:[(float(td[i][3]),td[i][2])]}
#print(trans_data.shape)
#for i in transitions.keys():
#print(i)
rewards = {}
rd = rwd_data
len_rewards = rd.shape[0]
for i in range(0,len_rewards):
rewards[rd[i][0]] = float(rd[i][1]) if rd[i][1] != 'None' else np.nan
#print(len(rewards.keys()))
rkeys = rewards.keys()
tkeys = transitions.keys()
st = ''
st2 = ''
for i in tkeys:
st += i + ' '
for j in rkeys:
st2 += j + ' '
print(st)
print(st2)
#This MDP class is to define the environment with which our agent is interacting

class MarkovDecisionProcess:
def __init__(self, states=[], transition={}, reward={}, gamma=0.9):
self.states = states
self.transition = transition
self.reward = reward
self.gamma = gamma

def Rwd(self, state):
return self.reward[state]

def Trans(self, state, action):
return self.transition[state][action]

def action(self, state):
return self.transition[state].keys()
Transitions = transitions
Rewards = rewards
States = transitions.keys()
#print(States)

mdp = MarkovDecisionProcess(states = States, transition = Transitions, reward = Rewards)
epsilon = 0.2
def val_iteration():
states = mdp.states
actions = mdp.action
Trans = mdp.Trans
Rwd = mdp.Rwd

#initializing the policy all with 0 value
V1 = {s: 0 for s in states}
while True:
V = V1.copy()
delta = 0

for s in states:
#Synchrounous Bellman update, updating the utility values
#two times or nested list comprehension done two times
V1[s] = Rwd(s) + gamma * max([sum([p*V[s1] for (p,s1) in Trans(s,a)]) for a in actions(s)])
#calculating the max difference in subsequent iterations
delta = max(delta, abs(V1[s]-V[s]))

if (delta < epsilon*(1-gamma)/gamma):
return V
def expected_utility(a,s,V):
Trans = mdp.Trans
return sum([p*V[s1] for (p,s1) in Trans(s,a)])
def best_policy(V):
states = mdp.states
actions = mdp.action
pi_policy = {}
for s in states:
pi_policy[s] = max(actions(s), key=lambda a: expected_utility(a,s,V))
return pi_policy
V = val_iteration()
print("State-Value")
for s in V:
print(s,' - ',V[s])
pi = best_policy(V)
print("n Optimal policy is n State - Action ")
for s in V:
print(s," ",'-'," ",pi[s])