Use data acquired by users #768
Conversation
|
PS: also a lot of clean-up still needed to pass linters.. |
| @@ -6,7 +6,8 @@ | |||
| import gym | |||
| import numpy as np | |||
| import torch as th | |||
| from stable_baselines3.common import policies, torch_layers | |||
| from stable_baselines3.common import policies, torch_layers, type_aliases | |||
| from stable_baselines3.common.vec_env import VecEnv | |||
There was a problem hiding this comment.
| from stable_baselines3.common.vec_env import VecEnv | |
| from stable_baselines3.common import vec_env |
Style guide https://google.github.io/styleguide/pyguide.html#22-imports
| np_obs = obs.detach().cpu().numpy() | ||
| np_actions = [] | ||
|
|
||
| for np_ob in np_obs: |
There was a problem hiding this comment.
Why do we iterate if we require num_envs == 1?
| return th_actions | ||
|
|
||
| @abc.abstractmethod | ||
| def _render(self, obs: np.ndarray) -> None: |
There was a problem hiding this comment.
Concrete implementations of this don't use obs -- do we need it?
| """Render the environment, optionally based on observation obs.""" | ||
|
|
||
| @abc.abstractmethod | ||
| def _query_action(self, obs: np.ndarray) -> np.ndarray: |
There was a problem hiding this comment.
Concrete implementations of this don't use obs -- do we need it?
| def _query_action(self, obs: np.ndarray) -> np.ndarray: | ||
| """Query human for an action, optionally based on observation obs.""" | ||
|
|
||
| def forward(self, *args): |
There was a problem hiding this comment.
Could inherit from HardCodedPolicy which already does this (code seems to be copied from there). Debatable whether it's "hard coded" (given it takes human input), but it's not a learned policy at least.
| user_input = input(self.action_prompt).strip().lower() | ||
| if user_input in self.action_map: | ||
| return np.array([self.action_map[user_input]]) | ||
| print("Invalid input. Try again.") |
There was a problem hiding this comment.
| print("Invalid input. Try again.") | |
| print(f"Invalid input '{input}. Try again.") |
| print("Invalid input. Try again.") | ||
|
|
||
|
|
||
| if __name__ == "__main__": |
There was a problem hiding this comment.
Consider moving example code into separate file
| @@ -0,0 +1,285 @@ | |||
| # type: ignore | |||
| # flake8: noqa | |||
There was a problem hiding this comment.
We'll want to fix these linting errors not just suppress them prior to merge.
| self._episode_returns - self._prev_episode_returns | ||
| ) | ||
| self._prev_episode_returns = self._episode_returns | ||
| mess = "steps={self._steps} episode_steps={self._episode_steps} episode_returns_delta={episode_returns_delta} episode_returns={self._episode_returns}".format( |
There was a problem hiding this comment.
Can use f-strings in modern Python instead of .format(**locals())
| from pyglet.window import key as keycodes | ||
|
|
||
|
|
||
| class Interactive(abc.ABC): |
There was a problem hiding this comment.
We probably want to refactor some of this Retro code before merging.
Description
This PR introduces a new abstract base policy
InteractivePolicyto support interactive data collection from human experts.There are two implementations for demo purposes:
TextInteractivePolicywhich supports simple Toy Text environments like FrozenLakeAtariInteractivePolicywhich is meant to support Atari games like Pong. This one is work in progress with many missing pieces. The idea is to rely on some code from retro-gym which is added in this PR with very slight modifications at src/imitation/policies/retro_gym_.pyLimitations
retro-gymenvironments. Maybe there are other options for demonstrating continuous-time data gathering?Testing
To explore the behavior of
TextInteractivePolicyyou can runpython src/imitation/policies/interactive_text.pyto observe a single episode rollout ofTextInteractivePolicyon FrozenLakepython examples/train_dagger_with_human_demos.pyto run a script collecting human demonstration data on FrozenLake and using it to train DAggerTo test that the interactive code taken from retro-gym is working you can
pip install gym-retroand then runpython src/imitation/policies/retro_gym_.pyThere are no demo scripts or tests for our wrapper
AtariInteractivePolicyyet since that is still WIP.