From 70eb229f366feb600a44dbba39823f01bd97bb73 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 1 Oct 2016 18:14:17 +0100 Subject: [PATCH 01/29] Modified specs to follow gym API --- rlenvs/Acrobot.lua | 26 ++++++++++++++++++++------ rlenvs/Atari.lua | 16 ++++++++++++++-- rlenvs/Blackjack.lua | 22 +++++++++++++++++----- rlenvs/CartPole.lua | 25 +++++++++++++++++++------ rlenvs/Catch.lua | 16 ++++++++++++++-- rlenvs/CliffWalking.lua | 20 ++++++++++++++++---- rlenvs/DynaMaze.lua | 20 ++++++++++++++++---- rlenvs/GridWorld.lua | 20 ++++++++++++++++---- rlenvs/JacksCarRental.lua | 22 +++++++++++++++++----- rlenvs/MountainCar.lua | 20 ++++++++++++++++---- rlenvs/MultiArmedBandit.lua | 6 +++++- rlenvs/RandomWalk.lua | 10 ++++++++-- rlenvs/Taxi.lua | 28 +++++++++++++++++++++------- rlenvs/WindyWorld.lua | 23 ++++++++++++++++++----- 14 files changed, 217 insertions(+), 57 deletions(-) diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index 38af8ff..8444c41 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -22,17 +22,30 @@ end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges function Acrobot:getStateSpec() - return { - {'real', 1, {-math.pi, math.pi}}, -- Joint 1 angle - {'real', 1, {-math.pi, math.pi}}, -- Joint 2 angle - {'real', 1, {-4*math.pi, 4*math.pi}}, -- Joint 1 angular velocity - {'real', 1, {-9*math.pi, 9*math.pi}} -- Joint 2 angular velocity + local state = {} + state['name'] = 'Box' + state['shape'] = {4} + state['low'] = { + -math.pi, -- Joint 1 angle + -math.pi, -- Joint 2 angle + -4*math.pi, -- Joint 1 angular velocity + -9*math.pi -- Joint 2 angular velocity } + state['high'] = { + math.pi, -- Joint 1 angle + math.pi, -- Joint 2 angle + 4*math.pi, -- Joint 1 angular velocity + 9*math.pi -- Joint 2 angular velocity + } + return state end -- 1 action required, of type 'int', of dimensionality 1, with second torque joint in {-1, 0, 1} function Acrobot:getActionSpec() - return {'int', 1, {-1, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 3 + return action end -- Min and max reward @@ -53,6 +66,7 @@ end -- Swings the pole via torque on second joint function Acrobot:step(action) + action = action - 1 -- rescale the action local reward = -1 local terminal = false diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 83e1ea8..95ee3c8 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -45,12 +45,24 @@ end -- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 function Atari:getStateSpec() - return {'real', {3, 210, 160}, {0, 1}} + local state = {} + state['name'] = 'Box' + state['shape'] = {3, 210, 160} + state['low'] = { + 0 + } + state['high'] = { + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 18 (max) function Atari:getActionSpec() - return {'int', 1, {1, #self.actions}} + local action = {} + action['name'] = 'Discrete' + action['n'] = #self.actions + return action end -- RGB screen of height 210 and width 160 diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index 7d436c4..d3e5936 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -13,16 +13,28 @@ end -- 2 states returned, of type 'int', of dimensionality 1, for the player sum, dealer's showing card, and player-usable ace function Blackjack:getStateSpec() - return { - {'int', 1, {2, 20}}, - {'int', 1, {1, 10}}, - {'int', 1, {0, 1}} + local state = {} + state['name'] = 'Box' + state['shape'] = {3} + state['low'] = { + 2, + 1, + 0 } + state['high'] = { + 20, + 10, + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, either stand or hit function Blackjack:getActionSpec() - return {'int', 1, {0, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 2 + return action end -- Min and max reward diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index 5b1e49b..c5c31f7 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -20,17 +20,30 @@ end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges function CartPole:getStateSpec() - return { - {'real', 1, {-2.4, 2.4}}, -- Cart position - {'real', 1, {nil, nil}}, -- Cart velocity - {'real', 1, {math.rad(-12), math.rad(12)}}, -- Pole angle - {'real', 1, {nil, nil}} -- Pole angular velocity + local state = {} + state['name'] = 'Box' + state['shape'] = {4} + state['low'] = { + -2.4, -- Cart position + math.huge, -- Cart velocity + math.rad(-12), -- Pole angle + math.huge -- Pole angular velocity } + state['high'] = { + 2.4, -- Cart position + math.huge, -- Cart velocity + math.rad(12), -- Pole angle + math.huge -- Pole angular velocity + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left, right) function CartPole:getActionSpec() - return {'int', 1, {0, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 2 + return action end -- Min and max reward diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 40ba990..49e9e63 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -29,12 +29,24 @@ end -- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 function Catch:getStateSpec() - return {'int', {1, self.size, self.size}, {0, 1}} + local state = {} + state['name'] = 'Box' + state['shape'] = {1, self.size, self.size} + state['low'] = { + 0 + } + state['high'] = { + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 2 function Catch:getActionSpec() - return {'int', 1, {0, 2}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 3 + return action end -- RGB screen of size self.size x self.size diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 41d029f..3bfba33 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -9,15 +9,26 @@ end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 function CliffWalking:getStateSpec() - return { - {'int', 1, {1, 12}}, -- x - {'int', 1, {1, 4}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 1, -- x + 1 -- y } + state['high'] = { + 12, -- x + 4 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (up|right|down|left) function CliffWalking:getActionSpec() - return {'int', 1, {1, 4}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- Min and max reward @@ -34,6 +45,7 @@ end -- Move up, right, down or left function CliffWalking:step(action) + action = action + 1 -- scale action local reward = -1 local terminal = false diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 42faaf0..40b5021 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -28,15 +28,26 @@ end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-9 and y is 1-6 function DynaMaze:getStateSpec() - return { - {'int', 1, {1, 9}}, -- x - {'int', 1, {1, 6}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 1, -- x + 1 -- y } + state['high'] = { + 9, -- x + 6 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 function DynaMaze:getActionSpec() - return {'int', 1, {1, 4}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- Min and max reward @@ -57,6 +68,7 @@ end -- Move up, right, down or left function DynaMaze:step(action) + action = action + 1 -- scale action local reward = 0 local terminal = false diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 66fba69..a9023e4 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -20,15 +20,26 @@ end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 function GridWorld:getStateSpec() - return { - {'real', 1, {0, 1}}, -- x - {'real', 1, {0, 1}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 0, -- x + 0 -- y } + state['high'] = { + 1, -- x + 1 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 function GridWorld:getActionSpec() - return {'int', 1, {1, 4}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- Min and max reward @@ -45,6 +56,7 @@ end -- Move up, right, down or left function GridWorld:step(action) + action = action + 1 -- scale action local terminal = false -- Move diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index fa9d3a3..23c14b1 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -23,15 +23,26 @@ end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars function JacksCarRental:getStateSpec() - return { - {'int', 1, {0, 20}}, -- Lot 1 - {'int', 1, {0, 20}} -- Lot 2 + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 0, -- Lot 1 + 0 -- Lot 2 } + state['high'] = { + 20, -- Lot 1 + 20 -- Lot 2 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between -5 and 5 (max 5 cars can be moved overnight) function JacksCarRental:getActionSpec() - return {'int', 1, {-5, 5}} -- Negative numbers indicate transferring cars from lot 2 to lot 1 + local action = {} + action['name'] = 'Discrete' + action['n'] = 10 + return action end -- Min and max reward @@ -49,6 +60,7 @@ end -- Acts out a day and night for Jack's Car Rental function JacksCarRental:step(action) + action = action - 5 -- scale action local reward = 0 -- Reward in $ -- Customers rent cars from lot 1 during the day @@ -78,7 +90,7 @@ function JacksCarRental:step(action) self.lot1 = self.lot1 - carsMoved self.lot2 = self.lot2 + carsMoved reward = reward - 2*carsMoved - elseif action < 0 then + elseif action < 0 then -- Negative numbers indicate transferring cars from lot 2 to lot 1 carsMoved = math.min(-action, self.lot2) carsMoved = math.min(carsMoved, 20 - self.lot1) -- Move cars diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index 8d62109..b453995 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -9,15 +9,26 @@ end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges function MountainCar:getStateSpec() - return { - {'real', 1, {-0.07, 0.07}}, -- Velocity - {'real', 1, {-1.2, 0.6}} -- Position + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + -0.07, -- Velocity + -1.2 -- Position } + state['high'] = { + 0.07, -- Velocity + 0.6 -- Position + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between -1 and 1 (left, neutral, right) function MountainCar:getActionSpec() - return {'int', 1, {-1, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 3 + return action end -- Min and max reward @@ -36,6 +47,7 @@ end -- Drives the car function MountainCar:step(action) + action = action - 1 -- scale action -- Calculate height local height = math.sin(3*self.position) diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index 9261221..1aa1640 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -25,7 +25,10 @@ end -- 1 action required, of type 'int', of dimensionality 1, of the number of arms function MultiArmedBandit:getActionSpec() - return {'int', 1, {1, self.nArms}} + local action = {} + action['name'] = 'Discrete' + action['n'] = self.nArms + return action end -- Min and max rewards unknown when sampling from distributions @@ -40,6 +43,7 @@ end -- Pulls an arm function MultiArmedBandit:step(action) + action = action + 1 -- scale action -- Sample for reward local reward = torch.normal(self.armMeans[action], 1) diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index 8594330..e3f4c77 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -9,12 +9,18 @@ end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) function RandomWalk:getStateSpec() - return {'int', 1, {0, 6}} -- Position + local state = {} + state['name'] = 'Discrete' + state['n'] = 6 + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left or right) function RandomWalk:getActionSpec() - return {'int', 1, {0, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 2 + return action end -- Min and max reward diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 61d0ba5..235e40d 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -25,18 +25,32 @@ end -- 4 states returned, of type 'int', of dimensionality 1, where x and y are 0-5, fuel is -1-12, passenger position is 1-5 and destination is 1-4 function Taxi:getStateSpec() - return { - {'int', 1, {0, 4}}, -- x - {'int', 1, {0, 4}}, -- y - {'int', 1, {-1, 12}}, -- Fuel - {'int', 1, {1, 5}}, -- Passenger location - {'int', 1, {1, 4}}, -- Destination TODO: Work out why there are apparently 5 destination states in the original paper + local state = {} + state['name'] = 'Box' + state['shape'] = {5} + state['low'] = { + 0, -- x + 0, -- y + -1, -- Fuel + 1, -- Passenger location + 1 -- Destination TODO: Work out why there are apparently 5 destination states in the original paper } + state['high'] = { + 4, -- x + 4, -- y + 12, -- Fuel + 5, -- Passenger location + 4 -- Destination + } + return state end -- 1 action required, of type 'int', of dimensionality 1, where 1-4 is move N, E, S, W, 5 is Pickup, 6 is Putdown and 7 is Fillup function Taxi:getActionSpec() - return {'int', 1, {1, 7}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 7 + return action end -- Min and max reward diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index 9cba841..e89c5bc 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -12,18 +12,30 @@ end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-10 and y is 1-7 function WindyWorld:getStateSpec() - return { - {'int', 1, {1, 10}}, -- x - {'int', 1, {1, 7}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {5} + state['low'] = { + 1, -- x + 1 -- y } + state['high'] = { + 10, -- x + 7 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (for standard) or 1 and 8 (for king) function WindyWorld:getActionSpec() + local action = {} + action['name'] = 'Discrete' if self.king then - return {'int', 1, {1, 8}} + action['n'] = 8 + return action else - return {'int', 1, {1, 4}} + action['n'] = 4 + return action end end @@ -41,6 +53,7 @@ end -- Move up, right, down or left function WindyWorld:step(action) + action = action + 1 -- scale action local terminal = false -- Move From a56188ce5964dd4079b2729cfdd15fb49ab7d8bb Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sun, 2 Oct 2016 13:01:51 +0100 Subject: [PATCH 02/29] Update function names --- README.md | 6 +++--- experiment.lua | 4 ++-- rlenvs/Acrobot.lua | 6 +++--- rlenvs/Atari.lua | 6 +++--- rlenvs/Blackjack.lua | 6 +++--- rlenvs/CartPole.lua | 6 +++--- rlenvs/Catch.lua | 6 +++--- rlenvs/CliffWalking.lua | 6 +++--- rlenvs/DynaMaze.lua | 6 +++--- rlenvs/Env.lua | 6 +++--- rlenvs/GridWorld.lua | 6 +++--- rlenvs/JacksCarRental.lua | 6 +++--- rlenvs/MountainCar.lua | 6 +++--- rlenvs/MultiArmedBandit.lua | 6 +++--- rlenvs/RandomWalk.lua | 6 +++--- rlenvs/Taxi.lua | 6 +++--- rlenvs/WindyWorld.lua | 6 +++--- 17 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 1f48fb9..7be57d8 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Starts a new episode in the environment and returns the first `observation`. May Performs a step in the environment using `action` (which may be a list - see below), and returns the `reward`, the `observation` of the state transitioned to, and a `terminal` flag. Optionally provides `actionTaken`, if the environment provides supervision in the form of the actual action taken by the agent in spite of the provided action. -### stateSpec = env:getStateSpec() +### stateSpec = env:stateSpace() Returns a state specification as a list with 3 elements: @@ -67,11 +67,11 @@ Returns a state specification as a list with 3 elements: If several states are returned, `stateSpec` is itself a list of state specifications. Ranges may use `nil` if unknown. -### actionSpec = env:getActionSpec() +### actionSpec = env:actionSpace() Returns an action specification, with the same structure as used for state specifications. -### minReward, maxReward = env:getRewardSpec() +### minReward, maxReward = env:rewardSpace() Returns the minimum and maximum rewards produced by the environment. Values may be `nil` if unknown. diff --git a/experiment.lua b/experiment.lua index a184662..ae045bf 100644 --- a/experiment.lua +++ b/experiment.lua @@ -6,8 +6,8 @@ local qt = pcall(require, 'qt') -- Initialise and start environment local env = Catch({level = 2}) -local stateSpec = env:getStateSpec() -local actionSpec = env:getActionSpec() +local stateSpec = env:stateSpace() +local actionSpec = env:actionSpace() local observation = env:start() local reward, terminal diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index 8444c41..21565a9 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -21,7 +21,7 @@ function Acrobot:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function Acrobot:getStateSpec() +function Acrobot:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -41,7 +41,7 @@ function Acrobot:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, with second torque joint in {-1, 0, 1} -function Acrobot:getActionSpec() +function Acrobot:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 @@ -49,7 +49,7 @@ function Acrobot:getActionSpec() end -- Min and max reward -function Acrobot:getRewardSpec() +function Acrobot:rewardSpace() return -1, 0 end diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 95ee3c8..67a414b 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -44,7 +44,7 @@ function Atari:_init(opts) end -- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 -function Atari:getStateSpec() +function Atari:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3, 210, 160} @@ -58,7 +58,7 @@ function Atari:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 18 (max) -function Atari:getActionSpec() +function Atari:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = #self.actions @@ -71,7 +71,7 @@ function Atari:getDisplaySpec() end -- Min and max reward (unknown) -function Atari:getRewardSpec() +function Atari:rewardSpace() return nil, nil end diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index d3e5936..cd5cec5 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -12,7 +12,7 @@ function Blackjack:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for the player sum, dealer's showing card, and player-usable ace -function Blackjack:getStateSpec() +function Blackjack:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3} @@ -30,7 +30,7 @@ function Blackjack:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, either stand or hit -function Blackjack:getActionSpec() +function Blackjack:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 @@ -38,7 +38,7 @@ function Blackjack:getActionSpec() end -- Min and max reward -function Blackjack:getRewardSpec() +function Blackjack:rewardSpace() return -1, 1 end diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index c5c31f7..c0bd6db 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -19,7 +19,7 @@ function CartPole:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function CartPole:getStateSpec() +function CartPole:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -39,7 +39,7 @@ function CartPole:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left, right) -function CartPole:getActionSpec() +function CartPole:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 @@ -47,7 +47,7 @@ function CartPole:getActionSpec() end -- Min and max reward -function CartPole:getRewardSpec() +function CartPole:rewardSpace() return -1, 0 end diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 49e9e63..6b2c51e 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -28,7 +28,7 @@ function Catch:_init(opts) end -- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 -function Catch:getStateSpec() +function Catch:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {1, self.size, self.size} @@ -42,7 +42,7 @@ function Catch:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 2 -function Catch:getActionSpec() +function Catch:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 @@ -55,7 +55,7 @@ function Catch:getDisplaySpec() end -- Min and max reward -function Catch:getRewardSpec() +function Catch:rewardSpace() return 0, 1 end diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 3bfba33..eaa1850 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -8,7 +8,7 @@ function CliffWalking:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 -function CliffWalking:getStateSpec() +function CliffWalking:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function CliffWalking:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (up|right|down|left) -function CliffWalking:getActionSpec() +function CliffWalking:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 @@ -32,7 +32,7 @@ function CliffWalking:getActionSpec() end -- Min and max reward -function CliffWalking:getRewardSpec() +function CliffWalking:rewardSpace() return -100, -1 end diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 40b5021..7fa19f0 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -27,7 +27,7 @@ function DynaMaze:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-9 and y is 1-6 -function DynaMaze:getStateSpec() +function DynaMaze:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -43,7 +43,7 @@ function DynaMaze:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function DynaMaze:getActionSpec() +function DynaMaze:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 @@ -51,7 +51,7 @@ function DynaMaze:getActionSpec() end -- Min and max reward -function DynaMaze:getRewardSpec() +function DynaMaze:rewardSpace() return 0, 1 end diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 262c5f4..2d4bc55 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -5,8 +5,8 @@ local Env = classic.class('Env') -- Denote interfaces Env:mustHave('start') Env:mustHave('step') -Env:mustHave('getStateSpec') -Env:mustHave('getActionSpec') -Env:mustHave('getRewardSpec') +Env:mustHave('stateSpace') +Env:mustHave('actionSpace') +Env:mustHave('rewardSpace') return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index a9023e4..7b10edb 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -19,7 +19,7 @@ function GridWorld:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 -function GridWorld:getStateSpec() +function GridWorld:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -35,7 +35,7 @@ function GridWorld:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function GridWorld:getActionSpec() +function GridWorld:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 @@ -43,7 +43,7 @@ function GridWorld:getActionSpec() end -- Min and max reward -function GridWorld:getRewardSpec() +function GridWorld:rewardSpace() return torch.min(self.world), 0 end diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index 23c14b1..d7b85ce 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -22,7 +22,7 @@ function JacksCarRental:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars -function JacksCarRental:getStateSpec() +function JacksCarRental:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -38,7 +38,7 @@ function JacksCarRental:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between -5 and 5 (max 5 cars can be moved overnight) -function JacksCarRental:getActionSpec() +function JacksCarRental:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 10 @@ -46,7 +46,7 @@ function JacksCarRental:getActionSpec() end -- Min and max reward -function JacksCarRental:getRewardSpec() +function JacksCarRental:rewardSpace() return 0, 200 end diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index b453995..cbce89e 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -8,7 +8,7 @@ function MountainCar:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges -function MountainCar:getStateSpec() +function MountainCar:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function MountainCar:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between -1 and 1 (left, neutral, right) -function MountainCar:getActionSpec() +function MountainCar:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 @@ -32,7 +32,7 @@ function MountainCar:getActionSpec() end -- Min and max reward -function MountainCar:getRewardSpec() +function MountainCar:rewardSpace() return -2, 0 -- As height = sin(3x) is between -1 and 1, and reward = height - 1 end diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index 1aa1640..dc0368a 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -19,12 +19,12 @@ function MultiArmedBandit:_init(opts) end -- No state (not a contextual bandit) -function MultiArmedBandit:getStateSpec() +function MultiArmedBandit:stateSpace() return nil end -- 1 action required, of type 'int', of dimensionality 1, of the number of arms -function MultiArmedBandit:getActionSpec() +function MultiArmedBandit:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = self.nArms @@ -32,7 +32,7 @@ function MultiArmedBandit:getActionSpec() end -- Min and max rewards unknown when sampling from distributions -function MultiArmedBandit:getRewardSpec() +function MultiArmedBandit:rewardSpace() return nil, nil end diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index e3f4c77..e198f41 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -8,7 +8,7 @@ function RandomWalk:_init(opts) end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) -function RandomWalk:getStateSpec() +function RandomWalk:stateSpace() local state = {} state['name'] = 'Discrete' state['n'] = 6 @@ -16,7 +16,7 @@ function RandomWalk:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left or right) -function RandomWalk:getActionSpec() +function RandomWalk:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 @@ -24,7 +24,7 @@ function RandomWalk:getActionSpec() end -- Min and max reward -function RandomWalk:getRewardSpec() +function RandomWalk:rewardSpace() return 0, 1 end diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 235e40d..4c8afd7 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -24,7 +24,7 @@ function Taxi:_init(opts) end -- 4 states returned, of type 'int', of dimensionality 1, where x and y are 0-5, fuel is -1-12, passenger position is 1-5 and destination is 1-4 -function Taxi:getStateSpec() +function Taxi:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -46,7 +46,7 @@ function Taxi:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, where 1-4 is move N, E, S, W, 5 is Pickup, 6 is Putdown and 7 is Fillup -function Taxi:getActionSpec() +function Taxi:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 7 @@ -54,7 +54,7 @@ function Taxi:getActionSpec() end -- Min and max reward -function Taxi:getRewardSpec() +function Taxi:rewardSpace() return -20, 20 end diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index e89c5bc..5487b28 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -11,7 +11,7 @@ function WindyWorld:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-10 and y is 1-7 -function WindyWorld:getStateSpec() +function WindyWorld:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -27,7 +27,7 @@ function WindyWorld:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (for standard) or 1 and 8 (for king) -function WindyWorld:getActionSpec() +function WindyWorld:actionSpace() local action = {} action['name'] = 'Discrete' if self.king then @@ -40,7 +40,7 @@ function WindyWorld:getActionSpec() end -- Min and max reward -function WindyWorld:getRewardSpec() +function WindyWorld:rewardSpace() return -1, -1 end From 1fe33fae4e476902eb98713f1d5b695d1f32086e Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sun, 2 Oct 2016 13:14:09 +0100 Subject: [PATCH 03/29] Experiment uses new specs --- experiment.lua | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/experiment.lua b/experiment.lua index ae045bf..b08b799 100644 --- a/experiment.lua +++ b/experiment.lua @@ -6,33 +6,32 @@ local qt = pcall(require, 'qt') -- Initialise and start environment local env = Catch({level = 2}) -local stateSpec = env:stateSpace() -local actionSpec = env:actionSpace() +local actionSpace = env:actionSpace() local observation = env:start() -local reward, terminal +local reward, terminal = 0, false local episodes, totalReward = 0, 0 -local nSteps = 1000 * (stateSpec[2][2] - 1) -- Run for 1000 episodes +local nEpisodes = 1000 -- Display local window = qt and image.display({image=observation, zoom=10}) -for i = 1, nSteps do - -- Pick random action and execute it - local action = torch.random(actionSpec[3][1], actionSpec[3][2]) - reward, observation, terminal = env:step(action) - totalReward = totalReward + reward +for i = 1, nEpisodes do + while not terminal do + -- Pick random action and execute it + local action = torch.random(0, actionSpace['n'] - 1) + reward, observation, terminal = env:step(action) + totalReward = totalReward + reward - -- Display - if qt then - image.display({image=observation, zoom=10, win=window}) + -- Display + if qt then + image.display({image=observation, zoom=10, win=window}) + end end - -- If game finished, start again - if terminal then - episodes = episodes + 1 - observation = env:start() - end + episodes = episodes + 1 + observation = env:start() + terminal = false end print('Episodes: ' .. episodes) print('Total Reward: ' .. totalReward) From 522b8a498a66f4c3d2d5697e172b013499e18684 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sun, 2 Oct 2016 13:25:59 +0100 Subject: [PATCH 04/29] Changed function calls to getters --- README.md | 4 ++-- experiment.lua | 4 ++-- rlenvs/Acrobot.lua | 4 ++-- rlenvs/Atari.lua | 4 ++-- rlenvs/Blackjack.lua | 4 ++-- rlenvs/CartPole.lua | 4 ++-- rlenvs/Catch.lua | 4 ++-- rlenvs/CliffWalking.lua | 4 ++-- rlenvs/DynaMaze.lua | 4 ++-- rlenvs/Env.lua | 4 ++-- rlenvs/GridWorld.lua | 4 ++-- rlenvs/JacksCarRental.lua | 4 ++-- rlenvs/MountainCar.lua | 4 ++-- rlenvs/MultiArmedBandit.lua | 4 ++-- rlenvs/RandomWalk.lua | 4 ++-- rlenvs/Taxi.lua | 4 ++-- rlenvs/WindyWorld.lua | 4 ++-- 17 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 7be57d8..75df8bb 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Starts a new episode in the environment and returns the first `observation`. May Performs a step in the environment using `action` (which may be a list - see below), and returns the `reward`, the `observation` of the state transitioned to, and a `terminal` flag. Optionally provides `actionTaken`, if the environment provides supervision in the form of the actual action taken by the agent in spite of the provided action. -### stateSpec = env:stateSpace() +### stateSpec = env:getStateSpace() Returns a state specification as a list with 3 elements: @@ -67,7 +67,7 @@ Returns a state specification as a list with 3 elements: If several states are returned, `stateSpec` is itself a list of state specifications. Ranges may use `nil` if unknown. -### actionSpec = env:actionSpace() +### actionSpec = env:getActionSpace() Returns an action specification, with the same structure as used for state specifications. diff --git a/experiment.lua b/experiment.lua index b08b799..de15861 100644 --- a/experiment.lua +++ b/experiment.lua @@ -6,7 +6,7 @@ local qt = pcall(require, 'qt') -- Initialise and start environment local env = Catch({level = 2}) -local actionSpace = env:actionSpace() +local getActionSpace = env:getActionSpace() local observation = env:start() local reward, terminal = 0, false @@ -19,7 +19,7 @@ local window = qt and image.display({image=observation, zoom=10}) for i = 1, nEpisodes do while not terminal do -- Pick random action and execute it - local action = torch.random(0, actionSpace['n'] - 1) + local action = torch.random(0, getActionSpace['n'] - 1) reward, observation, terminal = env:step(action) totalReward = totalReward + reward diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index 21565a9..fc3e219 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -21,7 +21,7 @@ function Acrobot:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function Acrobot:stateSpace() +function Acrobot:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -41,7 +41,7 @@ function Acrobot:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, with second torque joint in {-1, 0, 1} -function Acrobot:actionSpace() +function Acrobot:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 67a414b..d98866f 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -44,7 +44,7 @@ function Atari:_init(opts) end -- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 -function Atari:stateSpace() +function Atari:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3, 210, 160} @@ -58,7 +58,7 @@ function Atari:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 18 (max) -function Atari:actionSpace() +function Atari:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = #self.actions diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index cd5cec5..49640ea 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -12,7 +12,7 @@ function Blackjack:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for the player sum, dealer's showing card, and player-usable ace -function Blackjack:stateSpace() +function Blackjack:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3} @@ -30,7 +30,7 @@ function Blackjack:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, either stand or hit -function Blackjack:actionSpace() +function Blackjack:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index c0bd6db..c49edb1 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -19,7 +19,7 @@ function CartPole:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function CartPole:stateSpace() +function CartPole:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -39,7 +39,7 @@ function CartPole:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left, right) -function CartPole:actionSpace() +function CartPole:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 6b2c51e..6f93084 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -28,7 +28,7 @@ function Catch:_init(opts) end -- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 -function Catch:stateSpace() +function Catch:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {1, self.size, self.size} @@ -42,7 +42,7 @@ function Catch:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 2 -function Catch:actionSpace() +function Catch:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index eaa1850..04fca5f 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -8,7 +8,7 @@ function CliffWalking:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 -function CliffWalking:stateSpace() +function CliffWalking:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function CliffWalking:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (up|right|down|left) -function CliffWalking:actionSpace() +function CliffWalking:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 7fa19f0..91c4546 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -27,7 +27,7 @@ function DynaMaze:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-9 and y is 1-6 -function DynaMaze:stateSpace() +function DynaMaze:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -43,7 +43,7 @@ function DynaMaze:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function DynaMaze:actionSpace() +function DynaMaze:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 2d4bc55..e839125 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -5,8 +5,8 @@ local Env = classic.class('Env') -- Denote interfaces Env:mustHave('start') Env:mustHave('step') -Env:mustHave('stateSpace') -Env:mustHave('actionSpace') +Env:mustHave('getStateSpace') +Env:mustHave('getActionSpace') Env:mustHave('rewardSpace') return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 7b10edb..319c646 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -19,7 +19,7 @@ function GridWorld:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 -function GridWorld:stateSpace() +function GridWorld:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -35,7 +35,7 @@ function GridWorld:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function GridWorld:actionSpace() +function GridWorld:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index d7b85ce..bb0a526 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -22,7 +22,7 @@ function JacksCarRental:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars -function JacksCarRental:stateSpace() +function JacksCarRental:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -38,7 +38,7 @@ function JacksCarRental:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between -5 and 5 (max 5 cars can be moved overnight) -function JacksCarRental:actionSpace() +function JacksCarRental:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 10 diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index cbce89e..d3ed972 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -8,7 +8,7 @@ function MountainCar:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges -function MountainCar:stateSpace() +function MountainCar:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function MountainCar:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between -1 and 1 (left, neutral, right) -function MountainCar:actionSpace() +function MountainCar:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index dc0368a..850fd17 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -19,12 +19,12 @@ function MultiArmedBandit:_init(opts) end -- No state (not a contextual bandit) -function MultiArmedBandit:stateSpace() +function MultiArmedBandit:getStateSpace() return nil end -- 1 action required, of type 'int', of dimensionality 1, of the number of arms -function MultiArmedBandit:actionSpace() +function MultiArmedBandit:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = self.nArms diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index e198f41..21fa7e1 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -8,7 +8,7 @@ function RandomWalk:_init(opts) end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) -function RandomWalk:stateSpace() +function RandomWalk:getStateSpace() local state = {} state['name'] = 'Discrete' state['n'] = 6 @@ -16,7 +16,7 @@ function RandomWalk:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left or right) -function RandomWalk:actionSpace() +function RandomWalk:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 4c8afd7..104e211 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -24,7 +24,7 @@ function Taxi:_init(opts) end -- 4 states returned, of type 'int', of dimensionality 1, where x and y are 0-5, fuel is -1-12, passenger position is 1-5 and destination is 1-4 -function Taxi:stateSpace() +function Taxi:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -46,7 +46,7 @@ function Taxi:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, where 1-4 is move N, E, S, W, 5 is Pickup, 6 is Putdown and 7 is Fillup -function Taxi:actionSpace() +function Taxi:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 7 diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index 5487b28..a80b461 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -11,7 +11,7 @@ function WindyWorld:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-10 and y is 1-7 -function WindyWorld:stateSpace() +function WindyWorld:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -27,7 +27,7 @@ function WindyWorld:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (for standard) or 1 and 8 (for king) -function WindyWorld:actionSpace() +function WindyWorld:getActionSpace() local action = {} action['name'] = 'Discrete' if self.king then From 9e929219e37f6c66e08136e5ce63f4478e525710 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 19 Oct 2016 10:53:09 +0100 Subject: [PATCH 05/29] table with all rlenv envs --- rlenvs/init.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rlenvs/init.lua b/rlenvs/init.lua index e42d836..b36ffac 100644 --- a/rlenvs/init.lua +++ b/rlenvs/init.lua @@ -24,4 +24,10 @@ for k, v in pairs(rlenvs) do end end +local envs ={} +for k,_ in pairs(rlenvs) do + envs[#envs+1]=k +end +rlenvs.envs = envs + return rlenvs From 1c276f28d38d0001ff7c8bbfe60e544b8c1b6e18 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 21 Oct 2016 15:19:09 +0100 Subject: [PATCH 06/29] Update function calls, added new steps method, updated README --- README.md | 6 +++++- experiment.lua | 4 ++-- rlenvs/Acrobot.lua | 4 ++-- rlenvs/Atari.lua | 4 ++-- rlenvs/Blackjack.lua | 4 ++-- rlenvs/CartPole.lua | 8 ++++---- rlenvs/Catch.lua | 4 ++-- rlenvs/CliffWalking.lua | 4 ++-- rlenvs/DynaMaze.lua | 4 ++-- rlenvs/Env.lua | 16 ++++++++++++++-- rlenvs/GridWorld.lua | 4 ++-- rlenvs/JacksCarRental.lua | 4 ++-- rlenvs/MountainCar.lua | 4 ++-- rlenvs/MultiArmedBandit.lua | 4 ++-- rlenvs/RandomWalk.lua | 4 ++-- rlenvs/Taxi.lua | 4 ++-- rlenvs/WindyWorld.lua | 4 ++-- rlenvs/init.lua | 2 +- 18 files changed, 52 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 75df8bb..9df612c 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ local observation = env:start() **Note that the API is under development and may be subject to change** +### rlenvs.envs + +A table of all possible environments implemented in rlenvs. + ### observation = env:start([opts]) Starts a new episode in the environment and returns the first `observation`. May take `opts`. @@ -71,7 +75,7 @@ If several states are returned, `stateSpec` is itself a list of state specificat Returns an action specification, with the same structure as used for state specifications. -### minReward, maxReward = env:rewardSpace() +### minReward, maxReward = env:getRewardSpace() Returns the minimum and maximum rewards produced by the environment. Values may be `nil` if unknown. diff --git a/experiment.lua b/experiment.lua index de15861..1fc8630 100644 --- a/experiment.lua +++ b/experiment.lua @@ -1,6 +1,6 @@ local image = require 'image' -local Catch = require 'rlenvs/Catch' - +require 'rlenvs' +local Catch = require('rlenvs.Catch') -- Detect QT for image display local qt = pcall(require, 'qt') diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index fc3e219..d47a289 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -49,7 +49,7 @@ function Acrobot:getActionSpace() end -- Min and max reward -function Acrobot:rewardSpace() +function Acrobot:getRewardSpace() return -1, 0 end @@ -65,7 +65,7 @@ function Acrobot:start() end -- Swings the pole via torque on second joint -function Acrobot:step(action) +function Acrobot:_step(action) action = action - 1 -- rescale the action local reward = -1 local terminal = false diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index d98866f..166530c 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -71,7 +71,7 @@ function Atari:getDisplaySpec() end -- Min and max reward (unknown) -function Atari:rewardSpace() +function Atari:getRewardSpace() return nil, nil end @@ -89,7 +89,7 @@ function Atari:start() end -- Steps in a game -function Atari:step(action) +function Atari:_step(action) -- Map action index to action for game action = self.actions[action] diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index 49640ea..b745ce0 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -38,7 +38,7 @@ function Blackjack:getActionSpace() end -- Min and max reward -function Blackjack:rewardSpace() +function Blackjack:getRewardSpace() return -1, 1 end @@ -63,7 +63,7 @@ function Blackjack:start() end -- Player stands or hits -function Blackjack:step(action) +function Blackjack:_step(action) local reward = 0 local terminal = false diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index c49edb1..2f4d4f0 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -47,7 +47,7 @@ function CartPole:getActionSpace() end -- Min and max reward -function CartPole:rewardSpace() +function CartPole:getRewardSpace() return -1, 0 end @@ -63,7 +63,7 @@ function CartPole:start() end -- Drives the cart -function CartPole:step(action) +function CartPole:_step(action) -- Calculate acceleration local force = action == 1 and self.forceMagnitude or -self.forceMagnitude local cosTheta = math.cos(self.theta) @@ -79,10 +79,10 @@ function CartPole:step(action) self.thetaDot = self.thetaDot + self.tau * thetaDotDot -- Check failure (if cart reaches sides of track/pole tips too much) - local reward = 0 + local reward = 1 local terminal = false if self.x < -2.4 or self.x > 2.4 or self.theta < math.rad(-12) or self.theta > math.rad(12) then - reward = -1 + reward = 0 terminal = true end diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 6f93084..505017c 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -55,7 +55,7 @@ function Catch:getDisplaySpec() end -- Min and max reward -function Catch:rewardSpace() +function Catch:getRewardSpace() return 0, 1 end @@ -92,7 +92,7 @@ function Catch:start() end -- Steps in a game -function Catch:step(action) +function Catch:_step(action) -- Reward is 0 by default local reward = 0 diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 04fca5f..29a581a 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -32,7 +32,7 @@ function CliffWalking:getActionSpace() end -- Min and max reward -function CliffWalking:rewardSpace() +function CliffWalking:getRewardSpace() return -100, -1 end @@ -44,7 +44,7 @@ function CliffWalking:start() end -- Move up, right, down or left -function CliffWalking:step(action) +function CliffWalking:_step(action) action = action + 1 -- scale action local reward = -1 local terminal = false diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 91c4546..e6d8577 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -51,7 +51,7 @@ function DynaMaze:getActionSpace() end -- Min and max reward -function DynaMaze:rewardSpace() +function DynaMaze:getRewardSpace() return 0, 1 end @@ -67,7 +67,7 @@ function DynaMaze:start() end -- Move up, right, down or left -function DynaMaze:step(action) +function DynaMaze:_step(action) action = action + 1 -- scale action local reward = 0 local terminal = false diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index e839125..656336e 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -4,9 +4,21 @@ local Env = classic.class('Env') -- Denote interfaces Env:mustHave('start') -Env:mustHave('step') +Env:mustHave('_step') Env:mustHave('getStateSpace') Env:mustHave('getActionSpace') -Env:mustHave('rewardSpace') +Env:mustHave('getRewardSpace') + +function Env:step(action) + local reward, state, terminal = self:_step(action) + self.currentStep = self.currentStep == nil and 1 or self.currentStep + self.maxSteps = self.maxSteps == nil and 1000 or self.maxSteps + if self.currentStep == self.maxSteps then + terminal = true + self.currentStep = 0 + end + self.currentStep = self.currentStep + 1 + return reward, state, terminal +end return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 319c646..01e54e2 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -43,7 +43,7 @@ function GridWorld:getActionSpace() end -- Min and max reward -function GridWorld:rewardSpace() +function GridWorld:getRewardSpace() return torch.min(self.world), 0 end @@ -55,7 +55,7 @@ function GridWorld:start() end -- Move up, right, down or left -function GridWorld:step(action) +function GridWorld:_step(action) action = action + 1 -- scale action local terminal = false diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index bb0a526..615d028 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -46,7 +46,7 @@ function JacksCarRental:getActionSpace() end -- Min and max reward -function JacksCarRental:rewardSpace() +function JacksCarRental:getRewardSpace() return 0, 200 end @@ -59,7 +59,7 @@ function JacksCarRental:start() end -- Acts out a day and night for Jack's Car Rental -function JacksCarRental:step(action) +function JacksCarRental:_step(action) action = action - 5 -- scale action local reward = 0 -- Reward in $ diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index d3ed972..38e697c 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -32,7 +32,7 @@ function MountainCar:getActionSpace() end -- Min and max reward -function MountainCar:rewardSpace() +function MountainCar:getRewardSpace() return -2, 0 -- As height = sin(3x) is between -1 and 1, and reward = height - 1 end @@ -46,7 +46,7 @@ function MountainCar:start() end -- Drives the car -function MountainCar:step(action) +function MountainCar:_step(action) action = action - 1 -- scale action -- Calculate height local height = math.sin(3*self.position) diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index 850fd17..a25eb12 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -32,7 +32,7 @@ function MultiArmedBandit:getActionSpace() end -- Min and max rewards unknown when sampling from distributions -function MultiArmedBandit:rewardSpace() +function MultiArmedBandit:getRewardSpace() return nil, nil end @@ -42,7 +42,7 @@ function MultiArmedBandit:start() end -- Pulls an arm -function MultiArmedBandit:step(action) +function MultiArmedBandit:_step(action) action = action + 1 -- scale action -- Sample for reward local reward = torch.normal(self.armMeans[action], 1) diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index 21fa7e1..5f27cc2 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -24,7 +24,7 @@ function RandomWalk:getActionSpace() end -- Min and max reward -function RandomWalk:rewardSpace() +function RandomWalk:getRewardSpace() return 0, 1 end @@ -36,7 +36,7 @@ function RandomWalk:start() end -- Move left or right -function RandomWalk:step(action) +function RandomWalk:_step(action) local reward = 0 local terminal = false diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 104e211..b22a272 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -54,7 +54,7 @@ function Taxi:getActionSpace() end -- Min and max reward -function Taxi:rewardSpace() +function Taxi:getRewardSpace() return -20, 20 end @@ -104,7 +104,7 @@ function Taxi:validMove(action) end -- Move up, right, down or left -function Taxi:step(action) +function Taxi:_step(action) local reward = -1 local terminal = false diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index a80b461..e9637f2 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -40,7 +40,7 @@ function WindyWorld:getActionSpace() end -- Min and max reward -function WindyWorld:rewardSpace() +function WindyWorld:getRewardSpace() return -1, -1 end @@ -52,7 +52,7 @@ function WindyWorld:start() end -- Move up, right, down or left -function WindyWorld:step(action) +function WindyWorld:_step(action) action = action + 1 -- scale action local terminal = false diff --git a/rlenvs/init.lua b/rlenvs/init.lua index b36ffac..6dee04a 100644 --- a/rlenvs/init.lua +++ b/rlenvs/init.lua @@ -1,7 +1,7 @@ local rlenvs = {} -- Include environments -rlenvs.Env = require 'rlenvs/Env' +Env = require 'rlenvs/Env' rlenvs.Acrobot = require 'rlenvs/Acrobot' rlenvs.Atari = require 'rlenvs/Atari' rlenvs.Blackjack = require 'rlenvs/Blackjack' From 63cf6db1a3cdd99f6e3f585171a22fb8ea137393 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 24 Oct 2016 11:19:49 +0100 Subject: [PATCH 07/29] Added timestep limits, exposed start function added _start function to environments --- rlenvs/Acrobot.lua | 7 +++++-- rlenvs/Atari.lua | 6 +++++- rlenvs/Blackjack.lua | 4 +++- rlenvs/CartPole.lua | 7 +++++-- rlenvs/Catch.lua | 3 ++- rlenvs/CliffWalking.lua | 3 ++- rlenvs/DynaMaze.lua | 3 ++- rlenvs/Env.lua | 22 +++++++++++++++++++--- rlenvs/GridWorld.lua | 3 ++- rlenvs/JacksCarRental.lua | 3 ++- rlenvs/MountainCar.lua | 6 +++++- rlenvs/MultiArmedBandit.lua | 3 ++- rlenvs/RandomWalk.lua | 3 ++- rlenvs/Taxi.lua | 6 +++++- rlenvs/WindyWorld.lua | 3 ++- 15 files changed, 63 insertions(+), 19 deletions(-) diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index d47a289..368e114 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -1,11 +1,14 @@ local classic = require 'classic' local Acrobot, super = classic.class('Acrobot', Env) +Acrobot.timeStepLimit = 500 -- Constructor function Acrobot:_init(opts) opts = opts or {} - + opts.timeStepLimit = Acrobot.timeStepLimit + super._init(self, opts) + -- Constants self.g = opts.g or 9.8 self.m1 = opts.m1 or 1 -- Mass of link 1 @@ -54,7 +57,7 @@ function Acrobot:getRewardSpace() end -- Resets the cart -function Acrobot:start() +function Acrobot:_start() -- Reset angles and velocities self.q1 = 0 -- Joint 1 angle self.q2 = 0 -- Joint 2 angle diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 166530c..23cb39c 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -6,11 +6,15 @@ if not hasALEWrap then end local Atari, super = classic.class('Atari', Env) +Atari.timeStepLimit = 100000 -- Constructor function Atari:_init(opts) -- Create ALEWrap options from opts opts = opts or {} + opts.timeStepLimit = Atari.timeStepLimit + super._init(self, opts) + if opts.lifeLossTerminal == nil then opts.lifeLossTerminal = true end @@ -76,7 +80,7 @@ function Atari:getRewardSpace() end -- Starts a new game, possibly with a random number of no-ops -function Atari:start() +function Atari:_start() local screen, reward, terminal if self.gameEnv._random_starts > 0 then diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index b745ce0..b174455 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -7,6 +7,8 @@ local Blackjack, super = classic.class('Blackjack', Env) function Blackjack:_init(opts) opts = opts or {} + super._init(self, opts) + -- Create number-only suit self.suit = torch.Tensor({2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 11}) end @@ -43,7 +45,7 @@ function Blackjack:getRewardSpace() end -- Draw 2 cards for player and dealer -function Blackjack:start() +function Blackjack:_start() -- Shuffle deck self.deck = torch.cat({self.suit, self.suit, self.suit, self.suit}, 1):index(1, torch.randperm(52):long()) diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index 2f4d4f0..cb5b584 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -1,11 +1,14 @@ local classic = require 'classic' local CartPole, super = classic.class('CartPole', Env) +CartPole.timeStepLimit = 200 -- Constructor function CartPole:_init(opts) opts = opts or {} - + opts.timeStepLimit = CartPole.timeStepLimit + super._init(self, opts) + -- Constants self.gravity = opts.gravity or 9.8 self.cartMass = opts.cartMass or 1.0 @@ -52,7 +55,7 @@ function CartPole:getRewardSpace() end -- Resets the cart -function CartPole:start() +function CartPole:_start() -- Reset position, angle and velocities self.x = 0 -- Cart position (m) self.xDot = 0 -- Cart velocity diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 505017c..16b9b94 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -5,6 +5,7 @@ local Catch, super = classic.class('Catch', Env) -- Constructor function Catch:_init(opts) opts = opts or {} + super._init(self, opts) -- Difficulty level self.level = opts.level or 2 @@ -76,7 +77,7 @@ function Catch:redraw() end -- Starts new game -function Catch:start() +function Catch:_start() -- Reset player and ball self.player.x = math.ceil(self.size / 2) self.ball.x = torch.random(self.size) diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 29a581a..4200e4f 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -5,6 +5,7 @@ local CliffWalking, super = classic.class('CliffWalking', Env) -- Constructor function CliffWalking:_init(opts) opts = opts or {} + super._init(self, opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 @@ -37,7 +38,7 @@ function CliffWalking:getRewardSpace() end -- Reset position -function CliffWalking:start() +function CliffWalking:_start() self.position = {1, 1} return self.position diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index e6d8577..c1b9ddb 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -5,6 +5,7 @@ local DynaMaze, super = classic.class('DynaMaze', Env) -- Constructor function DynaMaze:_init(opts) opts = opts or {} + super._init(self, opts) -- Set change: none|blocking|shortcut self.change = opts.change or 'none' @@ -56,7 +57,7 @@ function DynaMaze:getRewardSpace() end -- Reset position -function DynaMaze:start() +function DynaMaze:_start() if self.change == 'none' then self.position = {1, 4} else diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 656336e..4608143 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -3,16 +3,27 @@ local classic = require 'classic' local Env = classic.class('Env') -- Denote interfaces -Env:mustHave('start') +Env:mustHave('_start') Env:mustHave('_step') Env:mustHave('getStateSpace') Env:mustHave('getActionSpace') Env:mustHave('getRewardSpace') +function Env:_init(opts) + if opts.timeStepLimit and opts.maxSteps then + self.maxSteps = math.min(opts.timeStepLimit, opts.maxSteps) + elseif opts.maxSteps then + self.maxSteps = opts.maxSteps + elseif opts.timeStepLimit then + self.maxSteps = opts.timeStepLimit + else + self.maxSteps = 1000 + end + self.currentStep = 1 +end + function Env:step(action) local reward, state, terminal = self:_step(action) - self.currentStep = self.currentStep == nil and 1 or self.currentStep - self.maxSteps = self.maxSteps == nil and 1000 or self.maxSteps if self.currentStep == self.maxSteps then terminal = true self.currentStep = 0 @@ -21,4 +32,9 @@ function Env:step(action) return reward, state, terminal end +function Env:start() + self.currentStep = 1 + return self:_start() +end + return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 01e54e2..ae4ba34 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -6,6 +6,7 @@ local GridWorld, super = classic.class('GridWorld', Env) -- Constructor function GridWorld:_init(opts) opts = opts or {} + super._init(self, opts) -- Cost of moving in world (discretized) self.world = torch.Tensor(101, 101):fill(-0.5) @@ -48,7 +49,7 @@ function GridWorld:getRewardSpace() end -- Reset position -function GridWorld:start() +function GridWorld:_start() self.position = {0.2, 0.4} return self.position diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index 615d028..55593ef 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -19,6 +19,7 @@ end -- Constructor function JacksCarRental:_init(opts) opts = opts or {} + super._init(self, opts) end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars @@ -51,7 +52,7 @@ function JacksCarRental:getRewardSpace() end -- Resets the cars to 10 at each lot -function JacksCarRental:start() +function JacksCarRental:_start() self.lot1 = 10 self.lot2 = 10 diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index 38e697c..fe1715c 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -1,10 +1,14 @@ local classic = require 'classic' local MountainCar, super = classic.class('MountainCar', Env) +MountainCar.timeStepLimit = 200 -- Constructor function MountainCar:_init(opts) opts = opts or {} + opts.timeStepLimit = MountainCar.timeStepLimit + + super._init(self, opts) end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges @@ -37,7 +41,7 @@ function MountainCar:getRewardSpace() end -- Resets the car -function MountainCar:start() +function MountainCar:_start() -- Reset position and velocity self.position = -0.5 self.velocity = 0 diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index a25eb12..d4e6c75 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -5,6 +5,7 @@ local MultiArmedBandit, super = classic.class('MultiArmedBandit', Env) -- Constructor function MultiArmedBandit:_init(opts) opts = opts or {} + super._init(self, opts) -- Restless bandits (with a Gaussian random walk) self.restless = opts.restless or false @@ -37,7 +38,7 @@ function MultiArmedBandit:getRewardSpace() end -- Does nothing (distributions do not reset) -function MultiArmedBandit:start() +function MultiArmedBandit:_start() return nil end diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index 5f27cc2..a1df9b2 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -5,6 +5,7 @@ local RandomWalk, super = classic.class('RandomWalk', Env) -- Constructor function RandomWalk:_init(opts) opts = opts or {} + super._init(self, opts) end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) @@ -29,7 +30,7 @@ function RandomWalk:getRewardSpace() end -- Reset position -function RandomWalk:start() +function RandomWalk:_start() self.position = 3 return self.position diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index b22a272..962b90c 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -12,10 +12,14 @@ local classic = require 'classic' --]] local Taxi, super = classic.class('Taxi', Env) +Taxi.timeStepLimit = 200 -- Constructor function Taxi:_init(opts) opts = opts or {} + opts.timeStepLimit = Taxi.timeStepLimit + + super._init(self, opts) -- Passenger positions (Red, Green, Blue, Yellow) self.rgbyPos = {{0, 4}, {4, 4}, {3, 0}, {0, 0}} @@ -59,7 +63,7 @@ function Taxi:getRewardSpace() end -- Reset position, fuel and passenger -function Taxi:start() +function Taxi:_start() -- Randomise position and fuel self.position = {torch.random(0, 4), torch.random(0, 4)} self.fuel = torch.random(5, 12) diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index e9637f2..00fdce1 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -5,6 +5,7 @@ local WindyWorld, super = classic.class('WindyWorld', Env) -- Constructor function WindyWorld:_init(opts) opts = opts or {} + super._init(self, opts) -- Allow king's moves (8 directions) self.king = opts.king or false @@ -45,7 +46,7 @@ function WindyWorld:getRewardSpace() end -- Reset position -function WindyWorld:start() +function WindyWorld:_start() self.position = {1, 4} return self.position From c20c8b84179e20d39f506ce128bcdc16a2820b2f Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 24 Oct 2016 17:55:35 +0100 Subject: [PATCH 08/29] Added render --- experiment.lua | 11 +++-------- rlenvs/Env.lua | 14 +++++++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/experiment.lua b/experiment.lua index 1fc8630..0882f29 100644 --- a/experiment.lua +++ b/experiment.lua @@ -1,11 +1,8 @@ -local image = require 'image' require 'rlenvs' local Catch = require('rlenvs.Catch') --- Detect QT for image display -local qt = pcall(require, 'qt') -- Initialise and start environment -local env = Catch({level = 2}) +local env = Catch({level = 2, render = true}) local getActionSpace = env:getActionSpace() local observation = env:start() @@ -14,7 +11,7 @@ local episodes, totalReward = 0, 0 local nEpisodes = 1000 -- Display -local window = qt and image.display({image=observation, zoom=10}) +env:render() for i = 1, nEpisodes do while not terminal do @@ -24,9 +21,7 @@ for i = 1, nEpisodes do totalReward = totalReward + reward -- Display - if qt then - image.display({image=observation, zoom=10, win=window}) - end + env:render() end episodes = episodes + 1 diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 4608143..5dab14f 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -19,6 +19,10 @@ function Env:_init(opts) else self.maxSteps = 1000 end + if opts.render then + require 'image' self.qt = pcall(require, 'qt') + if not self.qt then print('Was not able to load qt to render, are you using qlua to run the script?') end + end self.currentStep = 1 end @@ -34,7 +38,15 @@ end function Env:start() self.currentStep = 1 - return self:_start() + local obs = self:_start() + return obs +end + +function Env:render() + if self.qt and self.getDisplay then + self.window = self.window == nil and image.display({ image = self:getDisplay(), zoom = 10 }) or self.window + image.display({ image = self:getDisplay(), zoom = 10, win = self.window }) + end end return Env From a9ea21ee76a3769fe5d6f450981cb085f03f92ca Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 26 Oct 2016 10:41:56 +0100 Subject: [PATCH 09/29] Added zoom option --- rlenvs/Env.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 5dab14f..43817d8 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -23,6 +23,7 @@ function Env:_init(opts) require 'image' self.qt = pcall(require, 'qt') if not self.qt then print('Was not able to load qt to render, are you using qlua to run the script?') end end + self.zoom = opts.zoom or 1 self.currentStep = 1 end @@ -44,8 +45,8 @@ end function Env:render() if self.qt and self.getDisplay then - self.window = self.window == nil and image.display({ image = self:getDisplay(), zoom = 10 }) or self.window - image.display({ image = self:getDisplay(), zoom = 10, win = self.window }) + self.window = self.window == nil and image.display({ image = self:getDisplay(), zoom = self.zoom }) or self.window + image.display({ image = self:getDisplay(), zoom = self.zoom, win = self.window }) end end From 430a2b46467610741c0d8c30f08c63617e7c2d41 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 26 Oct 2016 13:18:15 +0100 Subject: [PATCH 10/29] Added zoom variable to experiment --- experiment.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment.lua b/experiment.lua index 0882f29..17d4cf4 100644 --- a/experiment.lua +++ b/experiment.lua @@ -2,7 +2,7 @@ require 'rlenvs' local Catch = require('rlenvs.Catch') -- Initialise and start environment -local env = Catch({level = 2, render = true}) +local env = Catch({level = 2, render = true, zoom = 10}) local getActionSpace = env:getActionSpace() local observation = env:start() From 358a8c7401a5618b57cf09b7f80d3fb517acfde4 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 26 Oct 2016 13:20:06 +0100 Subject: [PATCH 11/29] fixed variable name --- experiment.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiment.lua b/experiment.lua index 17d4cf4..89e37d3 100644 --- a/experiment.lua +++ b/experiment.lua @@ -3,7 +3,7 @@ local Catch = require('rlenvs.Catch') -- Initialise and start environment local env = Catch({level = 2, render = true, zoom = 10}) -local getActionSpace = env:getActionSpace() +local actionSpace = env:getActionSpace() local observation = env:start() local reward, terminal = 0, false @@ -16,7 +16,7 @@ env:render() for i = 1, nEpisodes do while not terminal do -- Pick random action and execute it - local action = torch.random(0, getActionSpace['n'] - 1) + local action = torch.random(0, actionSpace['n'] - 1) reward, observation, terminal = env:step(action) totalReward = totalReward + reward From 135bf69ba8b0cdaf116fb878ca60958012d768f7 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 1 Oct 2016 18:14:17 +0100 Subject: [PATCH 12/29] Modified specs to follow gym API --- rlenvs/Acrobot.lua | 26 ++++++++++++++++++++------ rlenvs/Atari.lua | 16 ++++++++++++++-- rlenvs/Blackjack.lua | 22 +++++++++++++++++----- rlenvs/CartPole.lua | 25 +++++++++++++++++++------ rlenvs/Catch.lua | 16 ++++++++++++++-- rlenvs/CliffWalking.lua | 20 ++++++++++++++++---- rlenvs/DynaMaze.lua | 20 ++++++++++++++++---- rlenvs/GridWorld.lua | 20 ++++++++++++++++---- rlenvs/JacksCarRental.lua | 22 +++++++++++++++++----- rlenvs/MountainCar.lua | 20 ++++++++++++++++---- rlenvs/MultiArmedBandit.lua | 6 +++++- rlenvs/RandomWalk.lua | 10 ++++++++-- rlenvs/Taxi.lua | 28 +++++++++++++++++++++------- rlenvs/WindyWorld.lua | 23 ++++++++++++++++++----- 14 files changed, 217 insertions(+), 57 deletions(-) diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index 38af8ff..8444c41 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -22,17 +22,30 @@ end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges function Acrobot:getStateSpec() - return { - {'real', 1, {-math.pi, math.pi}}, -- Joint 1 angle - {'real', 1, {-math.pi, math.pi}}, -- Joint 2 angle - {'real', 1, {-4*math.pi, 4*math.pi}}, -- Joint 1 angular velocity - {'real', 1, {-9*math.pi, 9*math.pi}} -- Joint 2 angular velocity + local state = {} + state['name'] = 'Box' + state['shape'] = {4} + state['low'] = { + -math.pi, -- Joint 1 angle + -math.pi, -- Joint 2 angle + -4*math.pi, -- Joint 1 angular velocity + -9*math.pi -- Joint 2 angular velocity } + state['high'] = { + math.pi, -- Joint 1 angle + math.pi, -- Joint 2 angle + 4*math.pi, -- Joint 1 angular velocity + 9*math.pi -- Joint 2 angular velocity + } + return state end -- 1 action required, of type 'int', of dimensionality 1, with second torque joint in {-1, 0, 1} function Acrobot:getActionSpec() - return {'int', 1, {-1, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 3 + return action end -- Min and max reward @@ -53,6 +66,7 @@ end -- Swings the pole via torque on second joint function Acrobot:step(action) + action = action - 1 -- rescale the action local reward = -1 local terminal = false diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 83e1ea8..95ee3c8 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -45,12 +45,24 @@ end -- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 function Atari:getStateSpec() - return {'real', {3, 210, 160}, {0, 1}} + local state = {} + state['name'] = 'Box' + state['shape'] = {3, 210, 160} + state['low'] = { + 0 + } + state['high'] = { + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 18 (max) function Atari:getActionSpec() - return {'int', 1, {1, #self.actions}} + local action = {} + action['name'] = 'Discrete' + action['n'] = #self.actions + return action end -- RGB screen of height 210 and width 160 diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index 7d436c4..d3e5936 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -13,16 +13,28 @@ end -- 2 states returned, of type 'int', of dimensionality 1, for the player sum, dealer's showing card, and player-usable ace function Blackjack:getStateSpec() - return { - {'int', 1, {2, 20}}, - {'int', 1, {1, 10}}, - {'int', 1, {0, 1}} + local state = {} + state['name'] = 'Box' + state['shape'] = {3} + state['low'] = { + 2, + 1, + 0 } + state['high'] = { + 20, + 10, + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, either stand or hit function Blackjack:getActionSpec() - return {'int', 1, {0, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 2 + return action end -- Min and max reward diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index 5b1e49b..c5c31f7 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -20,17 +20,30 @@ end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges function CartPole:getStateSpec() - return { - {'real', 1, {-2.4, 2.4}}, -- Cart position - {'real', 1, {nil, nil}}, -- Cart velocity - {'real', 1, {math.rad(-12), math.rad(12)}}, -- Pole angle - {'real', 1, {nil, nil}} -- Pole angular velocity + local state = {} + state['name'] = 'Box' + state['shape'] = {4} + state['low'] = { + -2.4, -- Cart position + math.huge, -- Cart velocity + math.rad(-12), -- Pole angle + math.huge -- Pole angular velocity } + state['high'] = { + 2.4, -- Cart position + math.huge, -- Cart velocity + math.rad(12), -- Pole angle + math.huge -- Pole angular velocity + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left, right) function CartPole:getActionSpec() - return {'int', 1, {0, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 2 + return action end -- Min and max reward diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 40ba990..49e9e63 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -29,12 +29,24 @@ end -- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 function Catch:getStateSpec() - return {'int', {1, self.size, self.size}, {0, 1}} + local state = {} + state['name'] = 'Box' + state['shape'] = {1, self.size, self.size} + state['low'] = { + 0 + } + state['high'] = { + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 2 function Catch:getActionSpec() - return {'int', 1, {0, 2}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 3 + return action end -- RGB screen of size self.size x self.size diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 41d029f..3bfba33 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -9,15 +9,26 @@ end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 function CliffWalking:getStateSpec() - return { - {'int', 1, {1, 12}}, -- x - {'int', 1, {1, 4}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 1, -- x + 1 -- y } + state['high'] = { + 12, -- x + 4 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (up|right|down|left) function CliffWalking:getActionSpec() - return {'int', 1, {1, 4}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- Min and max reward @@ -34,6 +45,7 @@ end -- Move up, right, down or left function CliffWalking:step(action) + action = action + 1 -- scale action local reward = -1 local terminal = false diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 42faaf0..40b5021 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -28,15 +28,26 @@ end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-9 and y is 1-6 function DynaMaze:getStateSpec() - return { - {'int', 1, {1, 9}}, -- x - {'int', 1, {1, 6}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 1, -- x + 1 -- y } + state['high'] = { + 9, -- x + 6 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 function DynaMaze:getActionSpec() - return {'int', 1, {1, 4}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- Min and max reward @@ -57,6 +68,7 @@ end -- Move up, right, down or left function DynaMaze:step(action) + action = action + 1 -- scale action local reward = 0 local terminal = false diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 66fba69..a9023e4 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -20,15 +20,26 @@ end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 function GridWorld:getStateSpec() - return { - {'real', 1, {0, 1}}, -- x - {'real', 1, {0, 1}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 0, -- x + 0 -- y } + state['high'] = { + 1, -- x + 1 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 function GridWorld:getActionSpec() - return {'int', 1, {1, 4}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- Min and max reward @@ -45,6 +56,7 @@ end -- Move up, right, down or left function GridWorld:step(action) + action = action + 1 -- scale action local terminal = false -- Move diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index fa9d3a3..23c14b1 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -23,15 +23,26 @@ end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars function JacksCarRental:getStateSpec() - return { - {'int', 1, {0, 20}}, -- Lot 1 - {'int', 1, {0, 20}} -- Lot 2 + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + 0, -- Lot 1 + 0 -- Lot 2 } + state['high'] = { + 20, -- Lot 1 + 20 -- Lot 2 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between -5 and 5 (max 5 cars can be moved overnight) function JacksCarRental:getActionSpec() - return {'int', 1, {-5, 5}} -- Negative numbers indicate transferring cars from lot 2 to lot 1 + local action = {} + action['name'] = 'Discrete' + action['n'] = 10 + return action end -- Min and max reward @@ -49,6 +60,7 @@ end -- Acts out a day and night for Jack's Car Rental function JacksCarRental:step(action) + action = action - 5 -- scale action local reward = 0 -- Reward in $ -- Customers rent cars from lot 1 during the day @@ -78,7 +90,7 @@ function JacksCarRental:step(action) self.lot1 = self.lot1 - carsMoved self.lot2 = self.lot2 + carsMoved reward = reward - 2*carsMoved - elseif action < 0 then + elseif action < 0 then -- Negative numbers indicate transferring cars from lot 2 to lot 1 carsMoved = math.min(-action, self.lot2) carsMoved = math.min(carsMoved, 20 - self.lot1) -- Move cars diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index 8d62109..b453995 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -9,15 +9,26 @@ end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges function MountainCar:getStateSpec() - return { - {'real', 1, {-0.07, 0.07}}, -- Velocity - {'real', 1, {-1.2, 0.6}} -- Position + local state = {} + state['name'] = 'Box' + state['shape'] = {2} + state['low'] = { + -0.07, -- Velocity + -1.2 -- Position } + state['high'] = { + 0.07, -- Velocity + 0.6 -- Position + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between -1 and 1 (left, neutral, right) function MountainCar:getActionSpec() - return {'int', 1, {-1, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 3 + return action end -- Min and max reward @@ -36,6 +47,7 @@ end -- Drives the car function MountainCar:step(action) + action = action - 1 -- scale action -- Calculate height local height = math.sin(3*self.position) diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index 9261221..1aa1640 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -25,7 +25,10 @@ end -- 1 action required, of type 'int', of dimensionality 1, of the number of arms function MultiArmedBandit:getActionSpec() - return {'int', 1, {1, self.nArms}} + local action = {} + action['name'] = 'Discrete' + action['n'] = self.nArms + return action end -- Min and max rewards unknown when sampling from distributions @@ -40,6 +43,7 @@ end -- Pulls an arm function MultiArmedBandit:step(action) + action = action + 1 -- scale action -- Sample for reward local reward = torch.normal(self.armMeans[action], 1) diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index 8594330..e3f4c77 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -9,12 +9,18 @@ end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) function RandomWalk:getStateSpec() - return {'int', 1, {0, 6}} -- Position + local state = {} + state['name'] = 'Discrete' + state['n'] = 6 + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left or right) function RandomWalk:getActionSpec() - return {'int', 1, {0, 1}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 2 + return action end -- Min and max reward diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 61d0ba5..235e40d 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -25,18 +25,32 @@ end -- 4 states returned, of type 'int', of dimensionality 1, where x and y are 0-5, fuel is -1-12, passenger position is 1-5 and destination is 1-4 function Taxi:getStateSpec() - return { - {'int', 1, {0, 4}}, -- x - {'int', 1, {0, 4}}, -- y - {'int', 1, {-1, 12}}, -- Fuel - {'int', 1, {1, 5}}, -- Passenger location - {'int', 1, {1, 4}}, -- Destination TODO: Work out why there are apparently 5 destination states in the original paper + local state = {} + state['name'] = 'Box' + state['shape'] = {5} + state['low'] = { + 0, -- x + 0, -- y + -1, -- Fuel + 1, -- Passenger location + 1 -- Destination TODO: Work out why there are apparently 5 destination states in the original paper } + state['high'] = { + 4, -- x + 4, -- y + 12, -- Fuel + 5, -- Passenger location + 4 -- Destination + } + return state end -- 1 action required, of type 'int', of dimensionality 1, where 1-4 is move N, E, S, W, 5 is Pickup, 6 is Putdown and 7 is Fillup function Taxi:getActionSpec() - return {'int', 1, {1, 7}} + local action = {} + action['name'] = 'Discrete' + action['n'] = 7 + return action end -- Min and max reward diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index 9cba841..e89c5bc 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -12,18 +12,30 @@ end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-10 and y is 1-7 function WindyWorld:getStateSpec() - return { - {'int', 1, {1, 10}}, -- x - {'int', 1, {1, 7}} -- y + local state = {} + state['name'] = 'Box' + state['shape'] = {5} + state['low'] = { + 1, -- x + 1 -- y } + state['high'] = { + 10, -- x + 7 -- y + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (for standard) or 1 and 8 (for king) function WindyWorld:getActionSpec() + local action = {} + action['name'] = 'Discrete' if self.king then - return {'int', 1, {1, 8}} + action['n'] = 8 + return action else - return {'int', 1, {1, 4}} + action['n'] = 4 + return action end end @@ -41,6 +53,7 @@ end -- Move up, right, down or left function WindyWorld:step(action) + action = action + 1 -- scale action local terminal = false -- Move From 2afdffe34fac8ba19fa636d7855b6916de3714ef Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sun, 2 Oct 2016 13:01:51 +0100 Subject: [PATCH 13/29] Update function names --- README.md | 6 +++--- experiment.lua | 4 ++-- rlenvs/Acrobot.lua | 6 +++--- rlenvs/Atari.lua | 6 +++--- rlenvs/Blackjack.lua | 6 +++--- rlenvs/CartPole.lua | 6 +++--- rlenvs/Catch.lua | 6 +++--- rlenvs/CliffWalking.lua | 6 +++--- rlenvs/DynaMaze.lua | 6 +++--- rlenvs/Env.lua | 6 +++--- rlenvs/GridWorld.lua | 6 +++--- rlenvs/JacksCarRental.lua | 6 +++--- rlenvs/MountainCar.lua | 6 +++--- rlenvs/MultiArmedBandit.lua | 6 +++--- rlenvs/RandomWalk.lua | 6 +++--- rlenvs/Taxi.lua | 6 +++--- rlenvs/WindyWorld.lua | 6 +++--- 17 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index f1c33f7..acc4752 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Starts a new episode in the environment and returns the first `observation`. May Performs a step in the environment using `action` (which may be a list - see below), and returns the `reward`, the `observation` of the state transitioned to, and a `terminal` flag. Optionally provides `actionTaken`, if the environment provides supervision in the form of the actual action taken by the agent in spite of the provided action. -### stateSpec = env:getStateSpec() +### stateSpec = env:stateSpace() Returns a state specification as a list with 3 elements: @@ -68,11 +68,11 @@ Returns a state specification as a list with 3 elements: If several states are returned, `stateSpec` is itself a list of state specifications. Ranges may use `nil` if unknown. -### actionSpec = env:getActionSpec() +### actionSpec = env:actionSpace() Returns an action specification, with the same structure as used for state specifications. -### minReward, maxReward = env:getRewardSpec() +### minReward, maxReward = env:rewardSpace() Returns the minimum and maximum rewards produced by the environment. Values may be `nil` if unknown. diff --git a/experiment.lua b/experiment.lua index a184662..ae045bf 100644 --- a/experiment.lua +++ b/experiment.lua @@ -6,8 +6,8 @@ local qt = pcall(require, 'qt') -- Initialise and start environment local env = Catch({level = 2}) -local stateSpec = env:getStateSpec() -local actionSpec = env:getActionSpec() +local stateSpec = env:stateSpace() +local actionSpec = env:actionSpace() local observation = env:start() local reward, terminal diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index 8444c41..21565a9 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -21,7 +21,7 @@ function Acrobot:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function Acrobot:getStateSpec() +function Acrobot:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -41,7 +41,7 @@ function Acrobot:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, with second torque joint in {-1, 0, 1} -function Acrobot:getActionSpec() +function Acrobot:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 @@ -49,7 +49,7 @@ function Acrobot:getActionSpec() end -- Min and max reward -function Acrobot:getRewardSpec() +function Acrobot:rewardSpace() return -1, 0 end diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 95ee3c8..67a414b 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -44,7 +44,7 @@ function Atari:_init(opts) end -- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 -function Atari:getStateSpec() +function Atari:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3, 210, 160} @@ -58,7 +58,7 @@ function Atari:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 18 (max) -function Atari:getActionSpec() +function Atari:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = #self.actions @@ -71,7 +71,7 @@ function Atari:getDisplaySpec() end -- Min and max reward (unknown) -function Atari:getRewardSpec() +function Atari:rewardSpace() return nil, nil end diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index d3e5936..cd5cec5 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -12,7 +12,7 @@ function Blackjack:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for the player sum, dealer's showing card, and player-usable ace -function Blackjack:getStateSpec() +function Blackjack:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3} @@ -30,7 +30,7 @@ function Blackjack:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, either stand or hit -function Blackjack:getActionSpec() +function Blackjack:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 @@ -38,7 +38,7 @@ function Blackjack:getActionSpec() end -- Min and max reward -function Blackjack:getRewardSpec() +function Blackjack:rewardSpace() return -1, 1 end diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index c5c31f7..c0bd6db 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -19,7 +19,7 @@ function CartPole:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function CartPole:getStateSpec() +function CartPole:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -39,7 +39,7 @@ function CartPole:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left, right) -function CartPole:getActionSpec() +function CartPole:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 @@ -47,7 +47,7 @@ function CartPole:getActionSpec() end -- Min and max reward -function CartPole:getRewardSpec() +function CartPole:rewardSpace() return -1, 0 end diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 49e9e63..6b2c51e 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -28,7 +28,7 @@ function Catch:_init(opts) end -- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 -function Catch:getStateSpec() +function Catch:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {1, self.size, self.size} @@ -42,7 +42,7 @@ function Catch:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 2 -function Catch:getActionSpec() +function Catch:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 @@ -55,7 +55,7 @@ function Catch:getDisplaySpec() end -- Min and max reward -function Catch:getRewardSpec() +function Catch:rewardSpace() return 0, 1 end diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 3bfba33..eaa1850 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -8,7 +8,7 @@ function CliffWalking:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 -function CliffWalking:getStateSpec() +function CliffWalking:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function CliffWalking:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (up|right|down|left) -function CliffWalking:getActionSpec() +function CliffWalking:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 @@ -32,7 +32,7 @@ function CliffWalking:getActionSpec() end -- Min and max reward -function CliffWalking:getRewardSpec() +function CliffWalking:rewardSpace() return -100, -1 end diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 40b5021..7fa19f0 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -27,7 +27,7 @@ function DynaMaze:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-9 and y is 1-6 -function DynaMaze:getStateSpec() +function DynaMaze:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -43,7 +43,7 @@ function DynaMaze:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function DynaMaze:getActionSpec() +function DynaMaze:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 @@ -51,7 +51,7 @@ function DynaMaze:getActionSpec() end -- Min and max reward -function DynaMaze:getRewardSpec() +function DynaMaze:rewardSpace() return 0, 1 end diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 262c5f4..2d4bc55 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -5,8 +5,8 @@ local Env = classic.class('Env') -- Denote interfaces Env:mustHave('start') Env:mustHave('step') -Env:mustHave('getStateSpec') -Env:mustHave('getActionSpec') -Env:mustHave('getRewardSpec') +Env:mustHave('stateSpace') +Env:mustHave('actionSpace') +Env:mustHave('rewardSpace') return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index a9023e4..7b10edb 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -19,7 +19,7 @@ function GridWorld:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 -function GridWorld:getStateSpec() +function GridWorld:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -35,7 +35,7 @@ function GridWorld:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function GridWorld:getActionSpec() +function GridWorld:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 @@ -43,7 +43,7 @@ function GridWorld:getActionSpec() end -- Min and max reward -function GridWorld:getRewardSpec() +function GridWorld:rewardSpace() return torch.min(self.world), 0 end diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index 23c14b1..d7b85ce 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -22,7 +22,7 @@ function JacksCarRental:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars -function JacksCarRental:getStateSpec() +function JacksCarRental:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -38,7 +38,7 @@ function JacksCarRental:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between -5 and 5 (max 5 cars can be moved overnight) -function JacksCarRental:getActionSpec() +function JacksCarRental:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 10 @@ -46,7 +46,7 @@ function JacksCarRental:getActionSpec() end -- Min and max reward -function JacksCarRental:getRewardSpec() +function JacksCarRental:rewardSpace() return 0, 200 end diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index b453995..cbce89e 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -8,7 +8,7 @@ function MountainCar:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges -function MountainCar:getStateSpec() +function MountainCar:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function MountainCar:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between -1 and 1 (left, neutral, right) -function MountainCar:getActionSpec() +function MountainCar:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 @@ -32,7 +32,7 @@ function MountainCar:getActionSpec() end -- Min and max reward -function MountainCar:getRewardSpec() +function MountainCar:rewardSpace() return -2, 0 -- As height = sin(3x) is between -1 and 1, and reward = height - 1 end diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index 1aa1640..dc0368a 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -19,12 +19,12 @@ function MultiArmedBandit:_init(opts) end -- No state (not a contextual bandit) -function MultiArmedBandit:getStateSpec() +function MultiArmedBandit:stateSpace() return nil end -- 1 action required, of type 'int', of dimensionality 1, of the number of arms -function MultiArmedBandit:getActionSpec() +function MultiArmedBandit:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = self.nArms @@ -32,7 +32,7 @@ function MultiArmedBandit:getActionSpec() end -- Min and max rewards unknown when sampling from distributions -function MultiArmedBandit:getRewardSpec() +function MultiArmedBandit:rewardSpace() return nil, nil end diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index e3f4c77..e198f41 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -8,7 +8,7 @@ function RandomWalk:_init(opts) end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) -function RandomWalk:getStateSpec() +function RandomWalk:stateSpace() local state = {} state['name'] = 'Discrete' state['n'] = 6 @@ -16,7 +16,7 @@ function RandomWalk:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left or right) -function RandomWalk:getActionSpec() +function RandomWalk:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 @@ -24,7 +24,7 @@ function RandomWalk:getActionSpec() end -- Min and max reward -function RandomWalk:getRewardSpec() +function RandomWalk:rewardSpace() return 0, 1 end diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 235e40d..4c8afd7 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -24,7 +24,7 @@ function Taxi:_init(opts) end -- 4 states returned, of type 'int', of dimensionality 1, where x and y are 0-5, fuel is -1-12, passenger position is 1-5 and destination is 1-4 -function Taxi:getStateSpec() +function Taxi:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -46,7 +46,7 @@ function Taxi:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, where 1-4 is move N, E, S, W, 5 is Pickup, 6 is Putdown and 7 is Fillup -function Taxi:getActionSpec() +function Taxi:actionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 7 @@ -54,7 +54,7 @@ function Taxi:getActionSpec() end -- Min and max reward -function Taxi:getRewardSpec() +function Taxi:rewardSpace() return -20, 20 end diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index e89c5bc..5487b28 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -11,7 +11,7 @@ function WindyWorld:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-10 and y is 1-7 -function WindyWorld:getStateSpec() +function WindyWorld:stateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -27,7 +27,7 @@ function WindyWorld:getStateSpec() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (for standard) or 1 and 8 (for king) -function WindyWorld:getActionSpec() +function WindyWorld:actionSpace() local action = {} action['name'] = 'Discrete' if self.king then @@ -40,7 +40,7 @@ function WindyWorld:getActionSpec() end -- Min and max reward -function WindyWorld:getRewardSpec() +function WindyWorld:rewardSpace() return -1, -1 end From 0e3b56148ce8a56c4356653fbd03304723aabf07 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sun, 2 Oct 2016 13:14:09 +0100 Subject: [PATCH 14/29] Experiment uses new specs --- experiment.lua | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/experiment.lua b/experiment.lua index ae045bf..b08b799 100644 --- a/experiment.lua +++ b/experiment.lua @@ -6,33 +6,32 @@ local qt = pcall(require, 'qt') -- Initialise and start environment local env = Catch({level = 2}) -local stateSpec = env:stateSpace() -local actionSpec = env:actionSpace() +local actionSpace = env:actionSpace() local observation = env:start() -local reward, terminal +local reward, terminal = 0, false local episodes, totalReward = 0, 0 -local nSteps = 1000 * (stateSpec[2][2] - 1) -- Run for 1000 episodes +local nEpisodes = 1000 -- Display local window = qt and image.display({image=observation, zoom=10}) -for i = 1, nSteps do - -- Pick random action and execute it - local action = torch.random(actionSpec[3][1], actionSpec[3][2]) - reward, observation, terminal = env:step(action) - totalReward = totalReward + reward +for i = 1, nEpisodes do + while not terminal do + -- Pick random action and execute it + local action = torch.random(0, actionSpace['n'] - 1) + reward, observation, terminal = env:step(action) + totalReward = totalReward + reward - -- Display - if qt then - image.display({image=observation, zoom=10, win=window}) + -- Display + if qt then + image.display({image=observation, zoom=10, win=window}) + end end - -- If game finished, start again - if terminal then - episodes = episodes + 1 - observation = env:start() - end + episodes = episodes + 1 + observation = env:start() + terminal = false end print('Episodes: ' .. episodes) print('Total Reward: ' .. totalReward) From 3a39f0cfe1f7f11d31515e61d65c1bb8d959b1fd Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sun, 2 Oct 2016 13:25:59 +0100 Subject: [PATCH 15/29] Changed function calls to getters --- README.md | 4 ++-- experiment.lua | 4 ++-- rlenvs/Acrobot.lua | 4 ++-- rlenvs/Atari.lua | 4 ++-- rlenvs/Blackjack.lua | 4 ++-- rlenvs/CartPole.lua | 4 ++-- rlenvs/Catch.lua | 4 ++-- rlenvs/CliffWalking.lua | 4 ++-- rlenvs/DynaMaze.lua | 4 ++-- rlenvs/Env.lua | 4 ++-- rlenvs/GridWorld.lua | 4 ++-- rlenvs/JacksCarRental.lua | 4 ++-- rlenvs/MountainCar.lua | 4 ++-- rlenvs/MultiArmedBandit.lua | 4 ++-- rlenvs/RandomWalk.lua | 4 ++-- rlenvs/Taxi.lua | 4 ++-- rlenvs/WindyWorld.lua | 4 ++-- 17 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index acc4752..980b0b5 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Starts a new episode in the environment and returns the first `observation`. May Performs a step in the environment using `action` (which may be a list - see below), and returns the `reward`, the `observation` of the state transitioned to, and a `terminal` flag. Optionally provides `actionTaken`, if the environment provides supervision in the form of the actual action taken by the agent in spite of the provided action. -### stateSpec = env:stateSpace() +### stateSpec = env:getStateSpace() Returns a state specification as a list with 3 elements: @@ -68,7 +68,7 @@ Returns a state specification as a list with 3 elements: If several states are returned, `stateSpec` is itself a list of state specifications. Ranges may use `nil` if unknown. -### actionSpec = env:actionSpace() +### actionSpec = env:getActionSpace() Returns an action specification, with the same structure as used for state specifications. diff --git a/experiment.lua b/experiment.lua index b08b799..de15861 100644 --- a/experiment.lua +++ b/experiment.lua @@ -6,7 +6,7 @@ local qt = pcall(require, 'qt') -- Initialise and start environment local env = Catch({level = 2}) -local actionSpace = env:actionSpace() +local getActionSpace = env:getActionSpace() local observation = env:start() local reward, terminal = 0, false @@ -19,7 +19,7 @@ local window = qt and image.display({image=observation, zoom=10}) for i = 1, nEpisodes do while not terminal do -- Pick random action and execute it - local action = torch.random(0, actionSpace['n'] - 1) + local action = torch.random(0, getActionSpace['n'] - 1) reward, observation, terminal = env:step(action) totalReward = totalReward + reward diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index 21565a9..fc3e219 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -21,7 +21,7 @@ function Acrobot:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function Acrobot:stateSpace() +function Acrobot:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -41,7 +41,7 @@ function Acrobot:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, with second torque joint in {-1, 0, 1} -function Acrobot:actionSpace() +function Acrobot:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 67a414b..d98866f 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -44,7 +44,7 @@ function Atari:_init(opts) end -- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 -function Atari:stateSpace() +function Atari:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3, 210, 160} @@ -58,7 +58,7 @@ function Atari:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 18 (max) -function Atari:actionSpace() +function Atari:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = #self.actions diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index cd5cec5..49640ea 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -12,7 +12,7 @@ function Blackjack:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for the player sum, dealer's showing card, and player-usable ace -function Blackjack:stateSpace() +function Blackjack:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {3} @@ -30,7 +30,7 @@ function Blackjack:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, either stand or hit -function Blackjack:actionSpace() +function Blackjack:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index c0bd6db..c49edb1 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -19,7 +19,7 @@ function CartPole:_init(opts) end -- 4 states returned, of type 'real', of dimensionality 1, with differing ranges -function CartPole:stateSpace() +function CartPole:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {4} @@ -39,7 +39,7 @@ function CartPole:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left, right) -function CartPole:actionSpace() +function CartPole:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 6b2c51e..6f93084 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -28,7 +28,7 @@ function Catch:_init(opts) end -- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 -function Catch:stateSpace() +function Catch:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {1, self.size, self.size} @@ -42,7 +42,7 @@ function Catch:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 2 -function Catch:actionSpace() +function Catch:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index eaa1850..04fca5f 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -8,7 +8,7 @@ function CliffWalking:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 -function CliffWalking:stateSpace() +function CliffWalking:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function CliffWalking:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (up|right|down|left) -function CliffWalking:actionSpace() +function CliffWalking:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 7fa19f0..91c4546 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -27,7 +27,7 @@ function DynaMaze:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-9 and y is 1-6 -function DynaMaze:stateSpace() +function DynaMaze:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -43,7 +43,7 @@ function DynaMaze:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function DynaMaze:actionSpace() +function DynaMaze:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 2d4bc55..e839125 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -5,8 +5,8 @@ local Env = classic.class('Env') -- Denote interfaces Env:mustHave('start') Env:mustHave('step') -Env:mustHave('stateSpace') -Env:mustHave('actionSpace') +Env:mustHave('getStateSpace') +Env:mustHave('getActionSpace') Env:mustHave('rewardSpace') return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 7b10edb..319c646 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -19,7 +19,7 @@ function GridWorld:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 -function GridWorld:stateSpace() +function GridWorld:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -35,7 +35,7 @@ function GridWorld:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 -function GridWorld:actionSpace() +function GridWorld:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 4 diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index d7b85ce..bb0a526 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -22,7 +22,7 @@ function JacksCarRental:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars -function JacksCarRental:stateSpace() +function JacksCarRental:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -38,7 +38,7 @@ function JacksCarRental:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between -5 and 5 (max 5 cars can be moved overnight) -function JacksCarRental:actionSpace() +function JacksCarRental:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 10 diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index cbce89e..d3ed972 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -8,7 +8,7 @@ function MountainCar:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges -function MountainCar:stateSpace() +function MountainCar:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {2} @@ -24,7 +24,7 @@ function MountainCar:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between -1 and 1 (left, neutral, right) -function MountainCar:actionSpace() +function MountainCar:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 3 diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index dc0368a..850fd17 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -19,12 +19,12 @@ function MultiArmedBandit:_init(opts) end -- No state (not a contextual bandit) -function MultiArmedBandit:stateSpace() +function MultiArmedBandit:getStateSpace() return nil end -- 1 action required, of type 'int', of dimensionality 1, of the number of arms -function MultiArmedBandit:actionSpace() +function MultiArmedBandit:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = self.nArms diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index e198f41..21fa7e1 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -8,7 +8,7 @@ function RandomWalk:_init(opts) end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) -function RandomWalk:stateSpace() +function RandomWalk:getStateSpace() local state = {} state['name'] = 'Discrete' state['n'] = 6 @@ -16,7 +16,7 @@ function RandomWalk:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 1 (left or right) -function RandomWalk:actionSpace() +function RandomWalk:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 2 diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 4c8afd7..104e211 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -24,7 +24,7 @@ function Taxi:_init(opts) end -- 4 states returned, of type 'int', of dimensionality 1, where x and y are 0-5, fuel is -1-12, passenger position is 1-5 and destination is 1-4 -function Taxi:stateSpace() +function Taxi:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -46,7 +46,7 @@ function Taxi:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, where 1-4 is move N, E, S, W, 5 is Pickup, 6 is Putdown and 7 is Fillup -function Taxi:actionSpace() +function Taxi:getActionSpace() local action = {} action['name'] = 'Discrete' action['n'] = 7 diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index 5487b28..a80b461 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -11,7 +11,7 @@ function WindyWorld:_init(opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-10 and y is 1-7 -function WindyWorld:stateSpace() +function WindyWorld:getStateSpace() local state = {} state['name'] = 'Box' state['shape'] = {5} @@ -27,7 +27,7 @@ function WindyWorld:stateSpace() end -- 1 action required, of type 'int', of dimensionality 1, between 1 and 4 (for standard) or 1 and 8 (for king) -function WindyWorld:actionSpace() +function WindyWorld:getActionSpace() local action = {} action['name'] = 'Discrete' if self.king then From 834c8373be3f5c25d95c036f081abdecd2312b6d Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 19 Oct 2016 10:53:09 +0100 Subject: [PATCH 16/29] table with all rlenv envs --- rlenvs/init.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rlenvs/init.lua b/rlenvs/init.lua index e42d836..b36ffac 100644 --- a/rlenvs/init.lua +++ b/rlenvs/init.lua @@ -24,4 +24,10 @@ for k, v in pairs(rlenvs) do end end +local envs ={} +for k,_ in pairs(rlenvs) do + envs[#envs+1]=k +end +rlenvs.envs = envs + return rlenvs From b6f086b9a8ba72fb12d28cd0d300aee2930dd13b Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 21 Oct 2016 15:19:09 +0100 Subject: [PATCH 17/29] Update function calls, added new steps method, updated README --- README.md | 6 +++++- experiment.lua | 4 ++-- rlenvs/Acrobot.lua | 4 ++-- rlenvs/Atari.lua | 4 ++-- rlenvs/Blackjack.lua | 4 ++-- rlenvs/CartPole.lua | 8 ++++---- rlenvs/Catch.lua | 4 ++-- rlenvs/CliffWalking.lua | 4 ++-- rlenvs/DynaMaze.lua | 4 ++-- rlenvs/Env.lua | 16 ++++++++++++++-- rlenvs/GridWorld.lua | 4 ++-- rlenvs/JacksCarRental.lua | 4 ++-- rlenvs/MountainCar.lua | 4 ++-- rlenvs/MultiArmedBandit.lua | 4 ++-- rlenvs/RandomWalk.lua | 4 ++-- rlenvs/Taxi.lua | 4 ++-- rlenvs/WindyWorld.lua | 4 ++-- rlenvs/init.lua | 2 +- 18 files changed, 52 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 980b0b5..25d5128 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,10 @@ local observation = env:start() **Note that the API is under development and may be subject to change** +### rlenvs.envs + +A table of all possible environments implemented in rlenvs. + ### observation = env:start([opts]) Starts a new episode in the environment and returns the first `observation`. May take `opts`. @@ -72,7 +76,7 @@ If several states are returned, `stateSpec` is itself a list of state specificat Returns an action specification, with the same structure as used for state specifications. -### minReward, maxReward = env:rewardSpace() +### minReward, maxReward = env:getRewardSpace() Returns the minimum and maximum rewards produced by the environment. Values may be `nil` if unknown. diff --git a/experiment.lua b/experiment.lua index de15861..1fc8630 100644 --- a/experiment.lua +++ b/experiment.lua @@ -1,6 +1,6 @@ local image = require 'image' -local Catch = require 'rlenvs/Catch' - +require 'rlenvs' +local Catch = require('rlenvs.Catch') -- Detect QT for image display local qt = pcall(require, 'qt') diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index fc3e219..d47a289 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -49,7 +49,7 @@ function Acrobot:getActionSpace() end -- Min and max reward -function Acrobot:rewardSpace() +function Acrobot:getRewardSpace() return -1, 0 end @@ -65,7 +65,7 @@ function Acrobot:start() end -- Swings the pole via torque on second joint -function Acrobot:step(action) +function Acrobot:_step(action) action = action - 1 -- rescale the action local reward = -1 local terminal = false diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index d98866f..166530c 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -71,7 +71,7 @@ function Atari:getDisplaySpec() end -- Min and max reward (unknown) -function Atari:rewardSpace() +function Atari:getRewardSpace() return nil, nil end @@ -89,7 +89,7 @@ function Atari:start() end -- Steps in a game -function Atari:step(action) +function Atari:_step(action) -- Map action index to action for game action = self.actions[action] diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index 49640ea..b745ce0 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -38,7 +38,7 @@ function Blackjack:getActionSpace() end -- Min and max reward -function Blackjack:rewardSpace() +function Blackjack:getRewardSpace() return -1, 1 end @@ -63,7 +63,7 @@ function Blackjack:start() end -- Player stands or hits -function Blackjack:step(action) +function Blackjack:_step(action) local reward = 0 local terminal = false diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index c49edb1..2f4d4f0 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -47,7 +47,7 @@ function CartPole:getActionSpace() end -- Min and max reward -function CartPole:rewardSpace() +function CartPole:getRewardSpace() return -1, 0 end @@ -63,7 +63,7 @@ function CartPole:start() end -- Drives the cart -function CartPole:step(action) +function CartPole:_step(action) -- Calculate acceleration local force = action == 1 and self.forceMagnitude or -self.forceMagnitude local cosTheta = math.cos(self.theta) @@ -79,10 +79,10 @@ function CartPole:step(action) self.thetaDot = self.thetaDot + self.tau * thetaDotDot -- Check failure (if cart reaches sides of track/pole tips too much) - local reward = 0 + local reward = 1 local terminal = false if self.x < -2.4 or self.x > 2.4 or self.theta < math.rad(-12) or self.theta > math.rad(12) then - reward = -1 + reward = 0 terminal = true end diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 6f93084..505017c 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -55,7 +55,7 @@ function Catch:getDisplaySpec() end -- Min and max reward -function Catch:rewardSpace() +function Catch:getRewardSpace() return 0, 1 end @@ -92,7 +92,7 @@ function Catch:start() end -- Steps in a game -function Catch:step(action) +function Catch:_step(action) -- Reward is 0 by default local reward = 0 diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 04fca5f..29a581a 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -32,7 +32,7 @@ function CliffWalking:getActionSpace() end -- Min and max reward -function CliffWalking:rewardSpace() +function CliffWalking:getRewardSpace() return -100, -1 end @@ -44,7 +44,7 @@ function CliffWalking:start() end -- Move up, right, down or left -function CliffWalking:step(action) +function CliffWalking:_step(action) action = action + 1 -- scale action local reward = -1 local terminal = false diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index 91c4546..e6d8577 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -51,7 +51,7 @@ function DynaMaze:getActionSpace() end -- Min and max reward -function DynaMaze:rewardSpace() +function DynaMaze:getRewardSpace() return 0, 1 end @@ -67,7 +67,7 @@ function DynaMaze:start() end -- Move up, right, down or left -function DynaMaze:step(action) +function DynaMaze:_step(action) action = action + 1 -- scale action local reward = 0 local terminal = false diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index e839125..656336e 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -4,9 +4,21 @@ local Env = classic.class('Env') -- Denote interfaces Env:mustHave('start') -Env:mustHave('step') +Env:mustHave('_step') Env:mustHave('getStateSpace') Env:mustHave('getActionSpace') -Env:mustHave('rewardSpace') +Env:mustHave('getRewardSpace') + +function Env:step(action) + local reward, state, terminal = self:_step(action) + self.currentStep = self.currentStep == nil and 1 or self.currentStep + self.maxSteps = self.maxSteps == nil and 1000 or self.maxSteps + if self.currentStep == self.maxSteps then + terminal = true + self.currentStep = 0 + end + self.currentStep = self.currentStep + 1 + return reward, state, terminal +end return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 319c646..01e54e2 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -43,7 +43,7 @@ function GridWorld:getActionSpace() end -- Min and max reward -function GridWorld:rewardSpace() +function GridWorld:getRewardSpace() return torch.min(self.world), 0 end @@ -55,7 +55,7 @@ function GridWorld:start() end -- Move up, right, down or left -function GridWorld:step(action) +function GridWorld:_step(action) action = action + 1 -- scale action local terminal = false diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index bb0a526..615d028 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -46,7 +46,7 @@ function JacksCarRental:getActionSpace() end -- Min and max reward -function JacksCarRental:rewardSpace() +function JacksCarRental:getRewardSpace() return 0, 200 end @@ -59,7 +59,7 @@ function JacksCarRental:start() end -- Acts out a day and night for Jack's Car Rental -function JacksCarRental:step(action) +function JacksCarRental:_step(action) action = action - 5 -- scale action local reward = 0 -- Reward in $ diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index d3ed972..38e697c 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -32,7 +32,7 @@ function MountainCar:getActionSpace() end -- Min and max reward -function MountainCar:rewardSpace() +function MountainCar:getRewardSpace() return -2, 0 -- As height = sin(3x) is between -1 and 1, and reward = height - 1 end @@ -46,7 +46,7 @@ function MountainCar:start() end -- Drives the car -function MountainCar:step(action) +function MountainCar:_step(action) action = action - 1 -- scale action -- Calculate height local height = math.sin(3*self.position) diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index 850fd17..a25eb12 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -32,7 +32,7 @@ function MultiArmedBandit:getActionSpace() end -- Min and max rewards unknown when sampling from distributions -function MultiArmedBandit:rewardSpace() +function MultiArmedBandit:getRewardSpace() return nil, nil end @@ -42,7 +42,7 @@ function MultiArmedBandit:start() end -- Pulls an arm -function MultiArmedBandit:step(action) +function MultiArmedBandit:_step(action) action = action + 1 -- scale action -- Sample for reward local reward = torch.normal(self.armMeans[action], 1) diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index 21fa7e1..5f27cc2 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -24,7 +24,7 @@ function RandomWalk:getActionSpace() end -- Min and max reward -function RandomWalk:rewardSpace() +function RandomWalk:getRewardSpace() return 0, 1 end @@ -36,7 +36,7 @@ function RandomWalk:start() end -- Move left or right -function RandomWalk:step(action) +function RandomWalk:_step(action) local reward = 0 local terminal = false diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index 104e211..b22a272 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -54,7 +54,7 @@ function Taxi:getActionSpace() end -- Min and max reward -function Taxi:rewardSpace() +function Taxi:getRewardSpace() return -20, 20 end @@ -104,7 +104,7 @@ function Taxi:validMove(action) end -- Move up, right, down or left -function Taxi:step(action) +function Taxi:_step(action) local reward = -1 local terminal = false diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index a80b461..e9637f2 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -40,7 +40,7 @@ function WindyWorld:getActionSpace() end -- Min and max reward -function WindyWorld:rewardSpace() +function WindyWorld:getRewardSpace() return -1, -1 end @@ -52,7 +52,7 @@ function WindyWorld:start() end -- Move up, right, down or left -function WindyWorld:step(action) +function WindyWorld:_step(action) action = action + 1 -- scale action local terminal = false diff --git a/rlenvs/init.lua b/rlenvs/init.lua index b36ffac..6dee04a 100644 --- a/rlenvs/init.lua +++ b/rlenvs/init.lua @@ -1,7 +1,7 @@ local rlenvs = {} -- Include environments -rlenvs.Env = require 'rlenvs/Env' +Env = require 'rlenvs/Env' rlenvs.Acrobot = require 'rlenvs/Acrobot' rlenvs.Atari = require 'rlenvs/Atari' rlenvs.Blackjack = require 'rlenvs/Blackjack' From 231953152c56c57d02c886cd4255ec98bf4bdfa7 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 24 Oct 2016 11:19:49 +0100 Subject: [PATCH 18/29] Added timestep limits, exposed start function added _start function to environments --- rlenvs/Acrobot.lua | 7 +++++-- rlenvs/Atari.lua | 6 +++++- rlenvs/Blackjack.lua | 4 +++- rlenvs/CartPole.lua | 7 +++++-- rlenvs/Catch.lua | 3 ++- rlenvs/CliffWalking.lua | 3 ++- rlenvs/DynaMaze.lua | 3 ++- rlenvs/Env.lua | 22 +++++++++++++++++++--- rlenvs/GridWorld.lua | 3 ++- rlenvs/JacksCarRental.lua | 3 ++- rlenvs/MountainCar.lua | 6 +++++- rlenvs/MultiArmedBandit.lua | 3 ++- rlenvs/RandomWalk.lua | 3 ++- rlenvs/Taxi.lua | 6 +++++- rlenvs/WindyWorld.lua | 3 ++- 15 files changed, 63 insertions(+), 19 deletions(-) diff --git a/rlenvs/Acrobot.lua b/rlenvs/Acrobot.lua index d47a289..368e114 100644 --- a/rlenvs/Acrobot.lua +++ b/rlenvs/Acrobot.lua @@ -1,11 +1,14 @@ local classic = require 'classic' local Acrobot, super = classic.class('Acrobot', Env) +Acrobot.timeStepLimit = 500 -- Constructor function Acrobot:_init(opts) opts = opts or {} - + opts.timeStepLimit = Acrobot.timeStepLimit + super._init(self, opts) + -- Constants self.g = opts.g or 9.8 self.m1 = opts.m1 or 1 -- Mass of link 1 @@ -54,7 +57,7 @@ function Acrobot:getRewardSpace() end -- Resets the cart -function Acrobot:start() +function Acrobot:_start() -- Reset angles and velocities self.q1 = 0 -- Joint 1 angle self.q2 = 0 -- Joint 2 angle diff --git a/rlenvs/Atari.lua b/rlenvs/Atari.lua index 166530c..23cb39c 100644 --- a/rlenvs/Atari.lua +++ b/rlenvs/Atari.lua @@ -6,11 +6,15 @@ if not hasALEWrap then end local Atari, super = classic.class('Atari', Env) +Atari.timeStepLimit = 100000 -- Constructor function Atari:_init(opts) -- Create ALEWrap options from opts opts = opts or {} + opts.timeStepLimit = Atari.timeStepLimit + super._init(self, opts) + if opts.lifeLossTerminal == nil then opts.lifeLossTerminal = true end @@ -76,7 +80,7 @@ function Atari:getRewardSpace() end -- Starts a new game, possibly with a random number of no-ops -function Atari:start() +function Atari:_start() local screen, reward, terminal if self.gameEnv._random_starts > 0 then diff --git a/rlenvs/Blackjack.lua b/rlenvs/Blackjack.lua index b745ce0..b174455 100644 --- a/rlenvs/Blackjack.lua +++ b/rlenvs/Blackjack.lua @@ -7,6 +7,8 @@ local Blackjack, super = classic.class('Blackjack', Env) function Blackjack:_init(opts) opts = opts or {} + super._init(self, opts) + -- Create number-only suit self.suit = torch.Tensor({2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 11}) end @@ -43,7 +45,7 @@ function Blackjack:getRewardSpace() end -- Draw 2 cards for player and dealer -function Blackjack:start() +function Blackjack:_start() -- Shuffle deck self.deck = torch.cat({self.suit, self.suit, self.suit, self.suit}, 1):index(1, torch.randperm(52):long()) diff --git a/rlenvs/CartPole.lua b/rlenvs/CartPole.lua index 2f4d4f0..cb5b584 100644 --- a/rlenvs/CartPole.lua +++ b/rlenvs/CartPole.lua @@ -1,11 +1,14 @@ local classic = require 'classic' local CartPole, super = classic.class('CartPole', Env) +CartPole.timeStepLimit = 200 -- Constructor function CartPole:_init(opts) opts = opts or {} - + opts.timeStepLimit = CartPole.timeStepLimit + super._init(self, opts) + -- Constants self.gravity = opts.gravity or 9.8 self.cartMass = opts.cartMass or 1.0 @@ -52,7 +55,7 @@ function CartPole:getRewardSpace() end -- Resets the cart -function CartPole:start() +function CartPole:_start() -- Reset position, angle and velocities self.x = 0 -- Cart position (m) self.xDot = 0 -- Cart velocity diff --git a/rlenvs/Catch.lua b/rlenvs/Catch.lua index 505017c..16b9b94 100644 --- a/rlenvs/Catch.lua +++ b/rlenvs/Catch.lua @@ -5,6 +5,7 @@ local Catch, super = classic.class('Catch', Env) -- Constructor function Catch:_init(opts) opts = opts or {} + super._init(self, opts) -- Difficulty level self.level = opts.level or 2 @@ -76,7 +77,7 @@ function Catch:redraw() end -- Starts new game -function Catch:start() +function Catch:_start() -- Reset player and ball self.player.x = math.ceil(self.size / 2) self.ball.x = torch.random(self.size) diff --git a/rlenvs/CliffWalking.lua b/rlenvs/CliffWalking.lua index 29a581a..4200e4f 100644 --- a/rlenvs/CliffWalking.lua +++ b/rlenvs/CliffWalking.lua @@ -5,6 +5,7 @@ local CliffWalking, super = classic.class('CliffWalking', Env) -- Constructor function CliffWalking:_init(opts) opts = opts or {} + super._init(self, opts) end -- 2 states returned, of type 'int', of dimensionality 1, where x is 1-12 and y is 1-4 @@ -37,7 +38,7 @@ function CliffWalking:getRewardSpace() end -- Reset position -function CliffWalking:start() +function CliffWalking:_start() self.position = {1, 1} return self.position diff --git a/rlenvs/DynaMaze.lua b/rlenvs/DynaMaze.lua index e6d8577..c1b9ddb 100644 --- a/rlenvs/DynaMaze.lua +++ b/rlenvs/DynaMaze.lua @@ -5,6 +5,7 @@ local DynaMaze, super = classic.class('DynaMaze', Env) -- Constructor function DynaMaze:_init(opts) opts = opts or {} + super._init(self, opts) -- Set change: none|blocking|shortcut self.change = opts.change or 'none' @@ -56,7 +57,7 @@ function DynaMaze:getRewardSpace() end -- Reset position -function DynaMaze:start() +function DynaMaze:_start() if self.change == 'none' then self.position = {1, 4} else diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 656336e..4608143 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -3,16 +3,27 @@ local classic = require 'classic' local Env = classic.class('Env') -- Denote interfaces -Env:mustHave('start') +Env:mustHave('_start') Env:mustHave('_step') Env:mustHave('getStateSpace') Env:mustHave('getActionSpace') Env:mustHave('getRewardSpace') +function Env:_init(opts) + if opts.timeStepLimit and opts.maxSteps then + self.maxSteps = math.min(opts.timeStepLimit, opts.maxSteps) + elseif opts.maxSteps then + self.maxSteps = opts.maxSteps + elseif opts.timeStepLimit then + self.maxSteps = opts.timeStepLimit + else + self.maxSteps = 1000 + end + self.currentStep = 1 +end + function Env:step(action) local reward, state, terminal = self:_step(action) - self.currentStep = self.currentStep == nil and 1 or self.currentStep - self.maxSteps = self.maxSteps == nil and 1000 or self.maxSteps if self.currentStep == self.maxSteps then terminal = true self.currentStep = 0 @@ -21,4 +32,9 @@ function Env:step(action) return reward, state, terminal end +function Env:start() + self.currentStep = 1 + return self:_start() +end + return Env diff --git a/rlenvs/GridWorld.lua b/rlenvs/GridWorld.lua index 01e54e2..ae4ba34 100644 --- a/rlenvs/GridWorld.lua +++ b/rlenvs/GridWorld.lua @@ -6,6 +6,7 @@ local GridWorld, super = classic.class('GridWorld', Env) -- Constructor function GridWorld:_init(opts) opts = opts or {} + super._init(self, opts) -- Cost of moving in world (discretized) self.world = torch.Tensor(101, 101):fill(-0.5) @@ -48,7 +49,7 @@ function GridWorld:getRewardSpace() end -- Reset position -function GridWorld:start() +function GridWorld:_start() self.position = {0.2, 0.4} return self.position diff --git a/rlenvs/JacksCarRental.lua b/rlenvs/JacksCarRental.lua index 615d028..55593ef 100644 --- a/rlenvs/JacksCarRental.lua +++ b/rlenvs/JacksCarRental.lua @@ -19,6 +19,7 @@ end -- Constructor function JacksCarRental:_init(opts) opts = opts or {} + super._init(self, opts) end -- 2 states returned, of type 'int', of dimensionality 1, for 0-20 cars @@ -51,7 +52,7 @@ function JacksCarRental:getRewardSpace() end -- Resets the cars to 10 at each lot -function JacksCarRental:start() +function JacksCarRental:_start() self.lot1 = 10 self.lot2 = 10 diff --git a/rlenvs/MountainCar.lua b/rlenvs/MountainCar.lua index 38e697c..fe1715c 100644 --- a/rlenvs/MountainCar.lua +++ b/rlenvs/MountainCar.lua @@ -1,10 +1,14 @@ local classic = require 'classic' local MountainCar, super = classic.class('MountainCar', Env) +MountainCar.timeStepLimit = 200 -- Constructor function MountainCar:_init(opts) opts = opts or {} + opts.timeStepLimit = MountainCar.timeStepLimit + + super._init(self, opts) end -- 2 states returned, of type 'real', of dimensionality 1, with differing ranges @@ -37,7 +41,7 @@ function MountainCar:getRewardSpace() end -- Resets the car -function MountainCar:start() +function MountainCar:_start() -- Reset position and velocity self.position = -0.5 self.velocity = 0 diff --git a/rlenvs/MultiArmedBandit.lua b/rlenvs/MultiArmedBandit.lua index a25eb12..d4e6c75 100644 --- a/rlenvs/MultiArmedBandit.lua +++ b/rlenvs/MultiArmedBandit.lua @@ -5,6 +5,7 @@ local MultiArmedBandit, super = classic.class('MultiArmedBandit', Env) -- Constructor function MultiArmedBandit:_init(opts) opts = opts or {} + super._init(self, opts) -- Restless bandits (with a Gaussian random walk) self.restless = opts.restless or false @@ -37,7 +38,7 @@ function MultiArmedBandit:getRewardSpace() end -- Does nothing (distributions do not reset) -function MultiArmedBandit:start() +function MultiArmedBandit:_start() return nil end diff --git a/rlenvs/RandomWalk.lua b/rlenvs/RandomWalk.lua index 5f27cc2..a1df9b2 100644 --- a/rlenvs/RandomWalk.lua +++ b/rlenvs/RandomWalk.lua @@ -5,6 +5,7 @@ local RandomWalk, super = classic.class('RandomWalk', Env) -- Constructor function RandomWalk:_init(opts) opts = opts or {} + super._init(self, opts) end -- 1 states returned, of type 'int', of dimensionality 1, between 0 and 6 (the terminal states) @@ -29,7 +30,7 @@ function RandomWalk:getRewardSpace() end -- Reset position -function RandomWalk:start() +function RandomWalk:_start() self.position = 3 return self.position diff --git a/rlenvs/Taxi.lua b/rlenvs/Taxi.lua index b22a272..962b90c 100644 --- a/rlenvs/Taxi.lua +++ b/rlenvs/Taxi.lua @@ -12,10 +12,14 @@ local classic = require 'classic' --]] local Taxi, super = classic.class('Taxi', Env) +Taxi.timeStepLimit = 200 -- Constructor function Taxi:_init(opts) opts = opts or {} + opts.timeStepLimit = Taxi.timeStepLimit + + super._init(self, opts) -- Passenger positions (Red, Green, Blue, Yellow) self.rgbyPos = {{0, 4}, {4, 4}, {3, 0}, {0, 0}} @@ -59,7 +63,7 @@ function Taxi:getRewardSpace() end -- Reset position, fuel and passenger -function Taxi:start() +function Taxi:_start() -- Randomise position and fuel self.position = {torch.random(0, 4), torch.random(0, 4)} self.fuel = torch.random(5, 12) diff --git a/rlenvs/WindyWorld.lua b/rlenvs/WindyWorld.lua index e9637f2..00fdce1 100644 --- a/rlenvs/WindyWorld.lua +++ b/rlenvs/WindyWorld.lua @@ -5,6 +5,7 @@ local WindyWorld, super = classic.class('WindyWorld', Env) -- Constructor function WindyWorld:_init(opts) opts = opts or {} + super._init(self, opts) -- Allow king's moves (8 directions) self.king = opts.king or false @@ -45,7 +46,7 @@ function WindyWorld:getRewardSpace() end -- Reset position -function WindyWorld:start() +function WindyWorld:_start() self.position = {1, 4} return self.position From b02b7603b0e35f7f545c7f86d9e5bd458074bdf3 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 24 Oct 2016 17:55:35 +0100 Subject: [PATCH 19/29] Added render --- experiment.lua | 11 +++-------- rlenvs/Env.lua | 14 +++++++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/experiment.lua b/experiment.lua index 1fc8630..0882f29 100644 --- a/experiment.lua +++ b/experiment.lua @@ -1,11 +1,8 @@ -local image = require 'image' require 'rlenvs' local Catch = require('rlenvs.Catch') --- Detect QT for image display -local qt = pcall(require, 'qt') -- Initialise and start environment -local env = Catch({level = 2}) +local env = Catch({level = 2, render = true}) local getActionSpace = env:getActionSpace() local observation = env:start() @@ -14,7 +11,7 @@ local episodes, totalReward = 0, 0 local nEpisodes = 1000 -- Display -local window = qt and image.display({image=observation, zoom=10}) +env:render() for i = 1, nEpisodes do while not terminal do @@ -24,9 +21,7 @@ for i = 1, nEpisodes do totalReward = totalReward + reward -- Display - if qt then - image.display({image=observation, zoom=10, win=window}) - end + env:render() end episodes = episodes + 1 diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 4608143..5dab14f 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -19,6 +19,10 @@ function Env:_init(opts) else self.maxSteps = 1000 end + if opts.render then + require 'image' self.qt = pcall(require, 'qt') + if not self.qt then print('Was not able to load qt to render, are you using qlua to run the script?') end + end self.currentStep = 1 end @@ -34,7 +38,15 @@ end function Env:start() self.currentStep = 1 - return self:_start() + local obs = self:_start() + return obs +end + +function Env:render() + if self.qt and self.getDisplay then + self.window = self.window == nil and image.display({ image = self:getDisplay(), zoom = 10 }) or self.window + image.display({ image = self:getDisplay(), zoom = 10, win = self.window }) + end end return Env From 68f773e2c09afda8c12a612757049fa439a3f3fa Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 26 Oct 2016 10:41:56 +0100 Subject: [PATCH 20/29] Added zoom option --- rlenvs/Env.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rlenvs/Env.lua b/rlenvs/Env.lua index 5dab14f..43817d8 100644 --- a/rlenvs/Env.lua +++ b/rlenvs/Env.lua @@ -23,6 +23,7 @@ function Env:_init(opts) require 'image' self.qt = pcall(require, 'qt') if not self.qt then print('Was not able to load qt to render, are you using qlua to run the script?') end end + self.zoom = opts.zoom or 1 self.currentStep = 1 end @@ -44,8 +45,8 @@ end function Env:render() if self.qt and self.getDisplay then - self.window = self.window == nil and image.display({ image = self:getDisplay(), zoom = 10 }) or self.window - image.display({ image = self:getDisplay(), zoom = 10, win = self.window }) + self.window = self.window == nil and image.display({ image = self:getDisplay(), zoom = self.zoom }) or self.window + image.display({ image = self:getDisplay(), zoom = self.zoom, win = self.window }) end end From 0f864338a6be1cd59624273978c784596125d40d Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 26 Oct 2016 13:18:15 +0100 Subject: [PATCH 21/29] Added zoom variable to experiment --- experiment.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment.lua b/experiment.lua index 0882f29..17d4cf4 100644 --- a/experiment.lua +++ b/experiment.lua @@ -2,7 +2,7 @@ require 'rlenvs' local Catch = require('rlenvs.Catch') -- Initialise and start environment -local env = Catch({level = 2, render = true}) +local env = Catch({level = 2, render = true, zoom = 10}) local getActionSpace = env:getActionSpace() local observation = env:start() From 904f747d2eb434628e7ded0965957d3bd8ba0737 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 26 Oct 2016 13:20:06 +0100 Subject: [PATCH 22/29] fixed variable name --- experiment.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiment.lua b/experiment.lua index 17d4cf4..89e37d3 100644 --- a/experiment.lua +++ b/experiment.lua @@ -3,7 +3,7 @@ local Catch = require('rlenvs.Catch') -- Initialise and start environment local env = Catch({level = 2, render = true, zoom = 10}) -local getActionSpace = env:getActionSpace() +local actionSpace = env:getActionSpace() local observation = env:start() local reward, terminal = 0, false @@ -16,7 +16,7 @@ env:render() for i = 1, nEpisodes do while not terminal do -- Pick random action and execute it - local action = torch.random(0, getActionSpace['n'] - 1) + local action = torch.random(0, actionSpace['n'] - 1) reward, observation, terminal = env:step(action) totalReward = totalReward + reward From c1f3b16fbe7f07d4bf6258dfc75c2c9ab24eeccf Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 9 Nov 2016 11:12:46 +0000 Subject: [PATCH 23/29] Modified XOWorld to new api standards --- rlenvs/XOWorld.lua | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/rlenvs/XOWorld.lua b/rlenvs/XOWorld.lua index 691d61c..c223aaa 100644 --- a/rlenvs/XOWorld.lua +++ b/rlenvs/XOWorld.lua @@ -102,24 +102,36 @@ function XOWorld:_init(opts) {67, 7}, {67, 27}, {67, 47}, {67, 67}} end --- 1 state returned, of type 'int', of dimensionality 1 x self.size x self.size, between 0 and 1 -function XOWorld:getStateSpec() - return {'int', {1, self.size, self.size}, {0, 1}} +-- 1 state returned, of type 'real', of dimensionality 3 x 210 x 160, between 0 and 1 +function XOWorld:getStateSpace() + local state = {} + state['name'] = 'Box' + state['shape'] = {3, self.size, self.size} + state['low'] = { + 0 + } + state['high'] = { + 1 + } + return state end -- 1 action required, of type 'int', of dimensionality 1, between 0 and 3 -function XOWorld:getActionSpec() - return {'int', 1, {0, 3}} +function XOWorld:getActionSpace() + local action = {} + action['name'] = 'Discrete' + action['n'] = 4 + return action end -- RGB screen of size self.size x self.size function XOWorld:getDisplaySpec() - return {'real', {3, self.size, self.size}, {0, 1}} + return {'real', {3, self.size, self.size}, {0, 1}} end -- Min and max reward -function XOWorld:getRewardSpec() - return -10, 10 +function XOWorld:getRewardSpace() + return -10, 10 end -- Redraws screen based on state and performs collision detection @@ -188,7 +200,7 @@ function XOWorld:update() end -- Starts new game -function XOWorld:start() +function XOWorld:_start() -- Reset time self.time = 1 @@ -257,7 +269,7 @@ function XOWorld:start() end -- Steps in a game -function XOWorld:step(action) +function XOWorld:_step(action) -- Move player if action == 0 then self.x = math.max(self.x - 1, 1) From 9114305a610e53abdf83077008309166f2fd4a91 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 9 Nov 2016 11:23:11 +0000 Subject: [PATCH 24/29] 1 channel for XOWorld --- rlenvs/XOWorld.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rlenvs/XOWorld.lua b/rlenvs/XOWorld.lua index c223aaa..bfeff99 100644 --- a/rlenvs/XOWorld.lua +++ b/rlenvs/XOWorld.lua @@ -106,7 +106,7 @@ end function XOWorld:getStateSpace() local state = {} state['name'] = 'Box' - state['shape'] = {3, self.size, self.size} + state['shape'] = {1, self.size, self.size} state['low'] = { 0 } From 796bc581fba9b385709e5ea26f468454cdb92f47 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 9 Nov 2016 11:33:11 +0000 Subject: [PATCH 25/29] Added super call --- rlenvs/XOWorld.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/rlenvs/XOWorld.lua b/rlenvs/XOWorld.lua index bfeff99..72f0ed9 100644 --- a/rlenvs/XOWorld.lua +++ b/rlenvs/XOWorld.lua @@ -6,6 +6,7 @@ local XOWorld, super = classic.class('XOWorld', Env) -- Constructor function XOWorld:_init(opts) opts = opts or {} + super._init(self, opts) -- Game mode (all circles, negative, or circles and crosses, negative and positive) self.double = opts.double or false From 52ad76d69559b81c4325e09e206248f39f765d08 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 14 Nov 2016 15:04:54 +0000 Subject: [PATCH 26/29] Added base tests --- tests/test.lua | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/test.lua diff --git a/tests/test.lua b/tests/test.lua new file mode 100644 index 0000000..671fbf1 --- /dev/null +++ b/tests/test.lua @@ -0,0 +1,21 @@ +local rlenvs = require 'rlenvs' + +local function runTest(env) + local Env = require('rlenvs.' .. env) + -- Initialise and start environment + local env = Env() + local actionSpace = env:getActionSpace() + local observation = env:start() + -- Pick random action and execute it + local action = torch.random(0, actionSpace['n'] - 1) + local reward, observation, terminal = env:step(action) + -- Display + env:render() +end + + +for index, env in ipairs(rlenvs.envs) do + if env ~= 'Atari' then + runTest(env) + end +end \ No newline at end of file From 59da7cd622e144cef8e53accd619e835142d6070 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 17 Nov 2016 10:06:06 +0000 Subject: [PATCH 27/29] Modified Minecraft functions to support api --- rlenvs/Minecraft.lua | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/rlenvs/Minecraft.lua b/rlenvs/Minecraft.lua index 8997462..b91dc66 100644 --- a/rlenvs/Minecraft.lua +++ b/rlenvs/Minecraft.lua @@ -16,6 +16,8 @@ end -- Constructor function Minecraft:_init(opts) + opts = opts or {} + super._init(self, opts) -- Check libaMalmoLua is available locally if not hasLibMalmoLua then print("Requires libMalmoLua.so in LUA_CPATH") @@ -120,16 +122,28 @@ function Minecraft:_init(opts) end -- 2 states returned, of type 'real', of dimensionality 1, from 0-1 -function Minecraft:getStateSpec() - return {'real', {3, self.height, self.width}, {0, 1}} +function Minecraft:getStateSpace() + local state = {} + state['name'] = 'Box' + state['shape'] = {3, self.height, self.width} + state['low'] = { + 0 + } + state['high'] = { + 1 + } + return state end -function Minecraft:getActionSpec() - return {'int', 1, {1, #self.actions}} +function Minecraft:getActionSpace() + local action = {} + action['name'] = 'Discrete' + action['n'] = #self.actions + return action end --- Min and max reward -function Minecraft:getRewardSpec() +-- Min and max reward (unknown) +function Minecraft:getRewardSpace() return nil, nil end @@ -158,7 +172,7 @@ function Minecraft:getRewards(world_rewards) end -- Reset position -function Minecraft:start() +function Minecraft:_start() local mission = MissionSpec(self.mission_xml, true) local mission_record = MissionRecordSpec() @@ -216,7 +230,7 @@ function Minecraft:start() end -- Move up, right, down or left -function Minecraft:step(action) +function Minecraft:_step(action) -- Do something local action = self.actions[action] self.agent_host:sendCommand(action) From 2d23ed7b227e5c63124a0ea26f2f00b3c65556fa Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 17 Nov 2016 10:23:31 +0000 Subject: [PATCH 28/29] Exclude minecraft from tests --- tests/test.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test.lua b/tests/test.lua index 671fbf1..c20fa8e 100644 --- a/tests/test.lua +++ b/tests/test.lua @@ -13,9 +13,8 @@ local function runTest(env) env:render() end - for index, env in ipairs(rlenvs.envs) do - if env ~= 'Atari' then + if env ~= 'Atari' and env ~= 'Minecraft' then runTest(env) end end \ No newline at end of file From 55f90eb6a84726fa1f886cd69f42d9b48a83fa68 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 25 Nov 2016 10:18:16 +0000 Subject: [PATCH 29/29] Added assertions to tests --- tests/test.lua | 53 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/tests/test.lua b/tests/test.lua index c20fa8e..3a31ce9 100644 --- a/tests/test.lua +++ b/tests/test.lua @@ -1,20 +1,41 @@ +require 'torch' local rlenvs = require 'rlenvs' -local function runTest(env) - local Env = require('rlenvs.' .. env) - -- Initialise and start environment - local env = Env() - local actionSpace = env:getActionSpace() - local observation = env:start() - -- Pick random action and execute it - local action = torch.random(0, actionSpace['n'] - 1) - local reward, observation, terminal = env:step(action) - -- Display - env:render() -end +local test = torch.TestSuite() +local tester + +function test.envs() + for index, env in ipairs(rlenvs.envs) do + local function runTest() + local Env = require('rlenvs.' .. env) + -- Initialise and start environment + local env = Env() + local actionSpace = env:getActionSpace() + local observation = env:start() + -- Pick random action and execute it + local action = torch.random(0, actionSpace['n'] - 1) + local reward, observation, terminal = env:step(action) + -- Display if implemented + env:render() + end -for index, env in ipairs(rlenvs.envs) do - if env ~= 'Atari' and env ~= 'Minecraft' then - runTest(env) + if env == 'Atari' then + local hasALEWrap = pcall(require, 'alewrap') + if not hasALEWrap then + tester:assert(pcall(runTest), 'Failed to run rlenv environment ' .. env) + end + elseif env == 'Minecraft' then + local hasSocket = pcall(require, 'socket') + local hasLibMalmoLua = pcall(require, 'libMalmoLua') + if not hasSocket and hasLibMalmoLua then + tester:assert(pcall(runTest), 'Failed to run rlenv environment ' .. env) + end + else + tester:assert(pcall(runTest), 'Failed to run rlenv environment ' .. env) + end end -end \ No newline at end of file +end + +tester = torch.Tester() +tester:add(test) +tester:run() \ No newline at end of file