Notes

#simple q learning env using openai gym(Frozenlake)

import gym
import numpy as np
def update_Q(Q, state, action, reward, next_state, alpha, gamma):
current_Q = Q[state, action]
max_next_Q = np.max(Q[next_state])
new_Q = current_Q + alpha * (reward + gamma * max_next_Q - current_Q)
Q[state, action] = new_Q
return Q

env = gym.make('FrozenLake-v1')
n_episodes = 10000
max_steps_per_episode = 100

alpha = 0.8
gamma = 0.95
epsilon = 0.2

Q = np.zeros((env.observation_space.n, env.action_space.n))

for episode in range(n_episodes):
state = env.reset()
done = False
step = 0

while not done and step < max_steps_per_episode:
# choose action using epsilon-greedy policy
if np.random.uniform() < epsilon:
action = env.action_space.sample()
else:
action = np.argmax(Q[state])

# take action and observe the resulting state and reward
next_state, reward, done, info = env.step(action)

# update Q-values
Q = update_Q(Q, state, action, reward, next_state, alpha, gamma)

# transition to the next state
state = next_state
step += 1

env.close()

#same as above but using SARSA insteat of using Q-learning
import gym
import numpy as np

# Create the environment
env = gym.make('Taxi-v3')

# Set hyperparameters
alpha = 0.5 # learning rate
gamma = 0.9 # discount factor
epsilon = 0.1 # epsilon-greedy parameter
num_episodes = 10000

# Initialize Q table
num_states = env.observation_space.n
num_actions = env.action_space.n
Q = np.zeros((num_states, num_actions))

# Run the SARSA algorithm
for i in range(num_episodes):
# Initialize episode
state = env.reset()
done = False

# Choose action using epsilon-greedy policy
if np.random.uniform() < epsilon:
action = env.action_space.sample()
else:
action = np.argmax(Q[state, :])

# Loop over steps in episode
while not done:
# Take the selected action and observe next state and reward
next_state, reward, done, info = env.step(action)

# Choose next action using epsilon-greedy policy
if np.random.uniform() < epsilon:
next_action = env.action_space.sample()
else:
next_action = np.argmax(Q[next_state, :])

# Update Q table
Q[state, action] += alpha * (reward + gamma * Q[next_state, next_action] - Q[state, action])

# Transition to next state and action
state = next_state
action = next_action

# Evaluate the agent
num_episodes = 1000
total_reward = 0

for i in range(num_episodes):
state = env.reset()
done = False
while not done:
action = np.argmax(Q[state, :])
state, reward, done, info = env.step(action)
total_reward += reward

avg_reward = total_reward / num_episodes
print("Average reward per episode: ", avg_reward)

#3-armed bandit problem with payout rate
import numpy as np

class Bandit:
def __init__(self, num_arms):
self.num_arms = num_arms
self.payout_rates = np.random.rand(num_arms)

def pull(self, arm):
return 1 if np.random.rand() < self.payout_rates[arm] else 0
num_arms = 3
bandit = Bandit(num_arms)

num_pulls = 1000
cumulative_rewards = np.zeros(num_arms)
num_pulls_per_arm = np.zeros(num_arms)

for t in range(num_pulls):
# Choose arm using epsilon-greedy policy
epsilon = 0.1
if np.random.rand() < epsilon:
arm = np.random.randint(num_arms)
else:
arm = np.argmax(cumulative_rewards / num_pulls_per_arm)

# Pull arm and update estimates
reward = bandit.pull(arm)
cumulative_rewards[arm] += reward
num_pulls_per_arm[arm] += 1

# Print estimated payout rates
print(cumulative_rewards / num_pulls_per_arm)