% Demonstration of DP/RL algorithms on the cleaning robot problem

%% Initialization

% common config
ccfg.problem = 'cleanrob_problem';
pparam.X = {0:5};           % state space
pparam.x_can = 5;           % location of garbage
pparam.rew_energy = 1;      % reward on power up
pparam.rew_can = 5;         % reward on collecting garbage
ccfg.model_params = {pparam};
ccfg.gamma = .5;            % discount factor
ccfg.eps = .001;            % default convergence criterion for qiter, piter algorithms
% some visualization config
ccfg.target = 'screen';   
ccfg.viscfg = struct;
ccfg.viscfg.figsize = [600 600];
ccfg.viscfg.pause = -1;

% retrieve (and if not available, silently compute in advance) optimal solution
cfg = ccfg;
cfg.datafile = 'cleanrob_optsol_pi';
if ~exist([cfg.datafile '.mat'], 'file'),
    cfg.run = 1;
    cfg.verb = -Inf;    % keep things silent
    piter(cfg);
end;
load(cfg.datafile, 'Q', 'h', 'Qh');
ccfg.viscfg.Qstar = Q;
ccfg.viscfg.hstar = h;
ccfg.viscfg.Qhstar = Qh;

% common parameters for online learning
onlcfg = ccfg;
onlcfg.reset = [2 3];   % reset the system to one of these states at each trial, randomly
onlcfg.explor = .3;     % (initial) exploration probability
onlcfg.explordecay = 1; % and decay rate
onlcfg.alpha = .2;      % learning rate (constant)
onlcfg.viscfg.pause = {'cfg.trial <= 1', -1; 'true', 0};


%% Q-iteration
cfg = ccfg;
cfg.visualize = 1;      % visualize top-level iters
cfg.run = 1;            % run the algo
qiter(cfg);


%% Policy iteration with Q-functions
cfg = ccfg;
cfg.visualize = 2;      % visualize also level-2 iters (namely, policy evaluation)
cfg.run = 1;
piter(cfg);

%% Monte Carlo policy iteration
cfg = onlcfg;
cfg.visualize = 2;
cfg.run = 1;
cfg.peval_trials = 10;      % how many trials to evaluate a policy for
cfg.visit = 'every';        % every-visit Monte Carlo, which is the version discussed in the lectures
cfg.explortype = 'step';    % explore at every step
cfg.resetreturns = 1;       % reset returns to zero for each policy, to obtain the version in the lectures               
cfg.h0 = -ones(1, 6);       % always go left   
cfg.randseed = 100;         % uncomment for repeatable results
cfg.viscfg.pause = {'cfg.mcstep && cfg.trial <= 2', -1; 'true', 0};
montecarlo(cfg);

%% Monte Carlo optimistic policy iteration 
cfg = onlcfg;
cfg.visualize = 2;
cfg.run = 1;
cfg.peval_trials = 1;       % policy improved after every trial => optimistic
cfg.visit = 'every';
cfg.explortype = 'step';
cfg.resetreturns = 0;       % don't reset the returns to zero this time
cfg.viscfg.pause = {'cfg.mctrial && cfg.trial <= 2', -1; 'true', 0};
% cfg.viscfg.pause = 0;
cfg.randseed = 10;          % uncomment for repeatable results
montecarlo(cfg);

%% SARSA
cfg = onlcfg;
cfg.visualize = 2;
cfg.run = 1;
cfg.randseed = 10;          % uncomment for repeatable results
cfg.eps = -1;               % don't set a convergence threshold, just run for the defined number of trials
sarsa(cfg);

%% Q-learning
cfg = onlcfg;
cfg.visualize = 2;
cfg.run = 1;
cfg.randseed = 10;          % uncomment for repeatable results
cfg.eps = -1;
qlearn(cfg);


