%% -------------------------------
clear all
probs = [0.7; 0.8; 0.9; 0];
payoff = [0; 10; 20; 30];
discount = 0.8;
payoffArray = [zeros(4,1), payoff];
% build transition matrices
transReset = [ones(4,1), zeros(4, 3)];
auxProbs = probs(1:(end-1)); % the last one is not relevant
transWait = diag([auxProbs;1]) + diag(1-auxProbs,1);
transArray(:,:,1) = transWait;
transArray(:,:,2) = transReset;
feasibleActions = {1, [1;2], [1:2], 2};
initialState = 1;
systemObj = systemClass(transArray, payoffArray,feasibleActions, initialState);

rng('default')
numSteps = 1000; 
%% ------------------------------------------
fprintf(1,'\n##  Q-learning run 1\n')
alpha = 0.1;
startQFactors = [ repmat(50,4,1), payoff(:)]; 
[QFactors, policy, visits] = QLearning(systemObj,'max',discount,numSteps, ...
    startQFactors,alpha);
display(startQFactors);
display(QFactors);
disp(['policy  ', num2str(policy')]);
display(visits);

%% ------------------------------------------
fprintf(1,'\n##  Q-learning run 2\n')
systemObj.setState(initialState);
startQFactors = [ repmat(900,4,1), payoff(:)]; 
[QFactors, policy] = QLearning(systemObj,'max',discount,numSteps, ...
    startQFactors,alpha);
display(startQFactors);
display(QFactors);
disp(['policy  ', num2str(policy')]);

%% ------------------------------------------
fprintf(1,'\n##  Q-learning run 3\n')
systemObj.setState(initialState);
alphaSpeedy = 0.5;
startQFactors = [ repmat(900,4,1), payoff(:)]; 
[QFactors, policy] = QLearning(systemObj,'max',discount,numSteps, ...
    startQFactors,alphaSpeedy);
display(startQFactors);
display(QFactors);
disp(['policy  ', num2str(policy')]);

%% ------------------------------------------
fprintf(1,'\n##  Q-learning run 4\n')
systemObj.setState(initialState);
startQFactors = [ repmat(1,4,1), repmat(0,4,1)]; 
[QFactors, policy, visits] = QLearning(systemObj,'max',discount,numSteps, ...
    startQFactors,alphaSpeedy);
display(startQFactors);
display(QFactors);
disp(['policy  ', num2str(policy')]);
display(visits);

%% ------------------------------------------
fprintf(1,'\n##  epsilonQ-learning run \n')
systemObj.setState(initialState);
numSteps = 10000; 
startQFactors = [ repmat(1,4,1), repmat(0,4,1)]; 
[QFactors, policy, visits] = QLearning(systemObj,'max',discount,numSteps, ...
    startQFactors,alpha,true,0.1);
display(startQFactors);
display(QFactors);
disp(['policy  ', num2str(policy')]);
display(visits);
