% control design for single MC 
% Jan. 2, 1997

clear all

% parameters

H=10;
my=3;                               % number of the states of y and u "my" 
rand('seed',39);
T=300;                              % similation length
T0=1;                               % memory length of tank system
randn('seed',13)
lambda=0.99;

% initial states

for t=1:T0
y(t)=1;
u(t)=2;
end
u(T0+1)=1;
sita=0.005*ones(my,my^2);          % transition probability matrix
n=zeros(my,my^2);                  % sufficient statistics
n0=sita;

% ideal system
    
    IS=(0.01/(my-1))*ones(my,my^2);
    IS(1,:)=0.99*ones(1,my^2);
    
% ideal controller
  
    IC=(1/my)*ones(my,my);
        
% optimal controller
  
   for t=T0+1:T-1
     [y(t),theta]=system(y,u,t,my);               % simulated system
     [n,sita]=pred(n0,y,u,t,my,n,lambda,sita);
     c=condes(sita,IS,IC,my,H);
%     u(t+1)=1;
     u(t+1)=contro(c,y,t,my);
   end
   
hold off

plot(y)