Notes

learning_rate=0.009;

state_steps1=7;
state_length1=state_steps1*4;

state_steps2=30;
state_length2=state_steps2*4;

state_steps3=7;
state_length3=state_steps3;


length_indicator1=0;
length_indicator2=0;
csv_length=1700;

%çiftlere biri teklere diğeri ?
state_memory=zeros(csv_length,state_length1);
state_prime_memory=zeros(csv_length,state_length1);

state_memory2=zeros(csv_length,state_length2);
state_prime_memory2=zeros(csv_length,state_length2);

dec_memory=zeros(csv_length,4);
dec_prime_memory=zeros(csv_length,4);

action_memory=zeros(csv_length,3);
reward_memory=zeros(csv_length,3);


is_first=2
if is_first==1
'1'
epsilon=1;

%fast network
fast_network=fitnet([128 128]);
fast_network.layers{1}.transferFcn='tansig';
fast_network.layers{2}.transferFcn='satlins';
%fast_network.layers{3}.transferFcn='satlins';



fast_network.trainFcn='traingd';
fast_network.trainParam.epochs=1;
fast_network.divideParam.trainRatio = 1;
fast_network.divideParam.valRatio = 0;
fast_network.divideParam.testRatio = 0;
fast_network.performFcn = 'mse';
fast_network.trainParam.min_grad=1e-30;
fast_network.trainParam.lr=0.00005;
fast_network.trainParam.showWindow = 0;

a=[rand(state_length1,2)];
x=2*rand-1;
b=[x -x;-x x];
[fast_network,tr1]=train(fast_network,a,b);


slow_network=fitnet([256 256]);
slow_network.layers{1}.transferFcn='tansig';
slow_network.layers{2}.transferFcn='satlins';
%slow_network.layers{3}.transferFcn='satlins';




slow_network.trainFcn='traingd';
slow_network.trainParam.epochs=1;
slow_network.divideParam.trainRatio = 1;
slow_network.divideParam.valRatio = 0;
slow_network.divideParam.testRatio = 0;
slow_network.performFcn = 'mse';
slow_network.trainParam.min_grad=1e-30;
slow_network.trainParam.lr=0.00005;
slow_network.trainParam.showWindow = 0;

a=[rand(state_length2,2)];
x=2*rand-1;
b=[x -x;-x x];
[slow_network,tr2]=train(slow_network,a,b);

volume_fast_network=patternnet([64 64]);
volume_fast_network.layers{1}.transferFcn='tansig';
volume_fast_network.layers{2}.transferFcn='satlins';
%dec_network.trainFcn='t';
volume_fast_network.trainParam.epochs=1;
volume_fast_network.divideParam.trainRatio = 1;
volume_fast_network.divideParam.valRatio = 0;
volume_fast_network.divideParam.testRatio = 0;
%dec_network.performFcn = 'mse';
volume_fast_network.trainParam.min_grad=1e-15;
volume_fast_network.trainParam.lr=0.00005;
volume_fast_network.trainParam.showWindow = 0;

a=[rand(state_length3,2)];
x=2*rand-1;
b=[x -x;-x x];

[volume_fast_network,tr3]=train(volume_fast_network,a,b);



elseif is_first==2
load fast_network
load slow_network
load volume_fast_network
'2';
end
csv_length=1700;
x=0;
states=csvread('btc_1h.csv',0,1,[0 1 0+csv_length 4]);
s_i=1;
for ii=1:4:((csv_length)*4) %duzelt

inputs(ii:(ii+3),1)=states(s_i,:);
s_i=s_i+1;

end
volumee=csvread('btc_1h.csv',0,5,[0 5 csv_length 5]);




iteration=csv_length-20;

ogren=0;
goster=1;
modee=2;
d_f=0.98;
for episode=1:1000

buy_memory=zeros(iteration+1,1);
sell_memory=zeros(iteration+1,1);

act_pos=0;
cash=1000;
amount=0;

islem_sayisi=0;


%3-2 for each time step
if ogren==0


for i=(130):(iteration-20)
for exp=1:3

selecter=rand;
if selecter <= 0.5 %düzelt
action_memory(i,exp)=1;
elseif selecter > 0.5 %düzelt
action_memory(i,exp)=2;
end

end
for a_i=1:3

if action_memory(i,a_i)==1 && ((a_i==1) || (a_i==3))
tgt=(states(i+1,4)+states(i+2,4)+states(i+3,4)+states(i+4,4))/4;
noww=(states(i+1,4)+states(i,4)+states(i-1,4))/3;
reward_memory(i,a_i)=log(tgt/noww)*100;

elseif action_memory(i,a_i)==2 && ((a_i==1) || (a_i==3))
noww=(states(i+1,4)+states(i,4)+states(i-1,4))/3;
tgt=(states(i+1,4)+states(i+2,4)+states(i+3,4)+states(i+4,4))/4;
reward_memory(i,a_i)=-log(tgt/noww)*100;
end

if action_memory(i,a_i)==1 && (a_i==2)
tgt=(states(i+1,4)+states(i+2,4)+states(i+3,4)+states(i+4,4)+states(i+5,4)+states(i+6)+states(i+7,4)+states(i+8,4)+states(i+9,4)+states(i+10,4))/10;
noww=(states(i+2,4)+states(i+1,4)+states(i,4)+states(i-1,4)+states(i-2,4))/5;
reward_memory(i,a_i)=log(tgt/noww)*100;
elseif action_memory(i,a_i)==2 && (a_i==2)
tgt=(states(i+1,4)+states(i+2,4)+states(i+3,4)+states(i+4,4)+states(i+5,4)+states(i+6)+states(i+7,4)+states(i+8,4)+states(i+9,4)+states(i+10,4))/10;
noww=(states(i+2,4)+states(i+1,4)+states(i,4)+states(i-1,4)+states(i-2,4))/5;
reward_memory(i,a_i)=-log(tgt/noww)*100;
end
end
randm=rand;
if i>132 && randm >0.875 %HER SEFERİNDE SADECE BİRİNİ EĞİTSEK ?
for t=1:3
if t==1
t_q=fast_network(normalisse(inputs(((i*4)-(state_length1-1):(i*4)))));
if action_memory(i-1,t)==1
loss=[reward_memory(i-1,t)+((max(t_q))*(0.95));
(max(t_q))*(0.95)];
elseif action_memory(i-1,t)==2
loss=[(max(t_q))*(0.95);
reward_memory(i-1,t)+((max(t_q))*(0.95))];
end
[fast_network,training1]=train(fast_network,normalisse(inputs((((i-1)*4)-(state_length1-1):((i-1)*4)))),loss);
elseif t==2
t_q=slow_network(normalisse(inputs(((i*4)-(state_length2-1):(i*4)))));
if action_memory(i-1,t)==1
loss=[reward_memory(i-1,t)+((max(t_q))*(0.95));
(max(t_q))*(0.95)];
elseif action_memory(i-1,t)==2
loss=[(max(t_q))*(0.95);
reward_memory(i-1,t)+((max(t_q))*(0.95))];
end
[slow_network,training2]=train(slow_network,normalisse(inputs((((i-1)*4)-(state_length2-1):((i-1)*4)))),loss);
elseif t==3
t_q=volume_fast_network(normalisse(volumee((i-(state_length3-1)):(i))));
if action_memory(i-1,t)==1
loss=[reward_memory(i-1,t)+((max(t_q))*(0.95));
(max(t_q))*(0.95)];
elseif action_memory(i-1,t)==2
loss=[(max(t_q))*(0.95);
reward_memory(i-1,t)+((max(t_q))*(0.95))];
end
[volume_fast_network,training]=train(volume_fast_network,normalisse(volumee((i-1)-(state_length3-1):(i-1))),loss);

end
end
end
end
end


if mod(episode,modee)==0

for i=130:(iteration-20)
test1=fast_network(normalisse(inputs(((i*4)-(state_length1-1):(i*4)))));
[action_val,action_idx1]=max(test1);
test2=slow_network(normalisse(inputs(((i*4)-(state_length2-1):(i*4)))));
[action_val,action_idx2]=max(test2);
test3=volume_fast_network(normalisse(volumee(i-(state_length3-1):i)));
[action_val,action_idx3]=max(test3);

if action_idx1==1 && action_idx2==1 && action_idx3==1 && act_pos < 1
islem_sayisi=islem_sayisi+1;
buy_memory(i)=i;
act_pos=1;
coin_price=states(i,4);
amount=cash/coin_price;
cash=cash-(amount*coin_price);

elseif action_idx1==2 && action_idx2==2 && action_idx3==2 && act_pos==1

islem_sayisi=islem_sayisi+1;
sell_memory(i)=i;
act_pos=-1;
coin_price=states(i,4);
cash=amount*coin_price;
amount=amount-(cash/coin_price);
end

end


end

if act_pos==1 && mod(episode,modee)==0
coin_price=states(i,4);
cash=amount*coin_price;
elseif act_pos==-1 && mod(episode,modee)==0
cash=cash;
end
if goster ==0 && mod(episode,modee)==0
if mod(episode,modee)==0
x=x+1;
scatter(x,cash,'b');
hold all
drawnow
save ('fast_network','fast_network')
save ('slow_network','slow_network')
save('volume_fast_network','volume_fast_network')
islem_sayisi
end

elseif goster==1 && mod(episode,modee)==0
save ('fast_network','fast_network')
save ('slow_network','slow_network')
save('volume_fast_network','volume_fast_network')
episode
cash
islem_sayisi
plot(states(1:iteration,4))
hold on
plot(buy_memory(1:iteration,1),states(1:iteration,4),'g*')
hold on
plot(sell_memory(1:iteration,1),states(1:iteration,4),'r*')
hold on

drawnow
elseif mod(episode,10)==0 && goster==2
save ('fast_network','fast_network')
save ('slow_network','slow_network')
save('volume_fast_network','volume_fast_network')
episode
end
end