Initial commit

README.md

@@ -8,16 +8,17 @@ tags:
 model-index:
 - name: PPO
   results:
-  - metrics:
-    - type: mean_reward
-      value: -336.89 +/- 406.36
-      name: mean_reward
-    task:
+  - task:
       type: reinforcement-learning
       name: reinforcement-learning
     dataset:
       name: Pendulum-v1
       type: Pendulum-v1
+    metrics:
+    - type: mean_reward
+      value: -189.25 +/- 66.36
+      name: mean_reward
+      verified: false
 ---
 
 # **PPO** Agent playing **Pendulum-v1**
@@ -35,17 +36,28 @@ RL Zoo: https://github.com/DLR-RM/rl-baselines3-zoo<br/>
 SB3: https://github.com/DLR-RM/stable-baselines3<br/>
 SB3 Contrib: https://github.com/Stable-Baselines-Team/stable-baselines3-contrib
 
+Install the RL Zoo (with SB3 and SB3-Contrib):
+```bash
+pip install rl_zoo3
+```
+
 ```
 # Download model and save it into the logs/ folder
-python -m utils.load_from_hub --algo ppo --env Pendulum-v1 -orga HumanCompatibleAI -f logs/
-python enjoy.py --algo ppo --env Pendulum-v1  -f logs/
+python -m rl_zoo3.load_from_hub --algo ppo --env Pendulum-v1 -orga HumanCompatibleAI -f logs/
+python -m rl_zoo3.enjoy --algo ppo --env Pendulum-v1  -f logs/
+```
+
+If you installed the RL Zoo3 via pip (`pip install rl_zoo3`), from anywhere you can do:
+```
+python -m rl_zoo3.load_from_hub --algo ppo --env Pendulum-v1 -orga HumanCompatibleAI -f logs/
+python -m rl_zoo3.enjoy --algo ppo --env Pendulum-v1  -f logs/
 ```
 
 ## Training (with the RL Zoo)
 ```
-python train.py --algo ppo --env Pendulum-v1 -f logs/
+python -m rl_zoo3.train --algo ppo --env Pendulum-v1 -f logs/
 # Upload the model and generate video (when possible)
-python -m utils.push_to_hub --algo ppo --env Pendulum-v1 -f logs/ -orga HumanCompatibleAI
+python -m rl_zoo3.push_to_hub --algo ppo --env Pendulum-v1 -f logs/ -orga HumanCompatibleAI
 ```
 
 ## Hyperparameters
@@ -64,3 +76,8 @@ OrderedDict([('clip_range', 0.2),
              ('use_sde', True),
              ('normalize', False)])
 ```
+
+# Environment Arguments
+```python
+{'render_mode': 'rgb_array'}
+```

args.yml

@@ -1,6 +1,8 @@
 !!python/object/apply:collections.OrderedDict
 - - - algo
     - ppo
+  - - conf_file
+    - null
   - - device
     - cpu
   - - env
@@ -8,7 +10,7 @@
   - - env_kwargs
     - null
   - - eval_episodes
-    - 5
+    - 0
   - - eval_freq
     - 25000
   - - gym_packages
@@ -16,7 +18,7 @@
   - - hyperparams
     - null
   - - log_folder
-    - pendulum_experts/seed_8/
+    - gymnasium_models
   - - log_interval
     - -1
   - - max_total_trials
@@ -36,11 +38,13 @@
   - - no_optim_plots
     - false
   - - num_threads
-    - 2
+    - 4
   - - optimization_log_path
     - null
   - - optimize_hyperparameters
     - false
+  - - progress
+    - false
   - - pruner
     - median
   - - sampler
@@ -50,15 +54,15 @@
   - - save_replay_buffer
     - false
   - - seed
-    - 8
+    - 3755125310
   - - storage
     - null
   - - study_name
     - null
   - - tensorboard_log
-    - runs/Pendulum-v1__ppo__8__1659105255
+    - ''
   - - track
-    - true
+    - false
   - - trained_agent
     - ''
   - - truncate_last_trajectory
@@ -72,4 +76,6 @@
   - - wandb_entity
     - null
   - - wandb_project_name
-    - pendulum_experts
+    - sb3
+  - - wandb_tags
+    - []

env_kwargs.yml

@@ -1 +1 @@
-{}
+render_mode: rgb_array

ppo-Pendulum-v1.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cbe8f9c58a0397a68c46a17fe7e7df5ea9f72f7bf8d24bf6c3cdf90a793ed6b
-size 142033
+oid sha256:d2a5572df0c6d6022519615b76c9e6ca5bce0e7d5cf859ec145cbf99a437b8b0
+size 137832

ppo-Pendulum-v1/_stable_baselines3_version

@@ -1 +1 @@
-1.6.0
+2.2.0a3

ppo-Pendulum-v1/data

@@ -3,65 +3,35 @@
         ":type:": "<class 'abc.ABCMeta'>",
         ":serialized:": "gAWVOwAAAAAAAACMIXN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi5wb2xpY2llc5SMEUFjdG9yQ3JpdGljUG9saWN5lJOULg==",
         "__module__": "stable_baselines3.common.policies",
-        "__doc__": "\n    Policy class for actor-critic algorithms (has both policy and value prediction).\n    Used by A2C, PPO and the likes.\n\n    :param observation_space: Observation space\n    :param action_space: Action space\n    :param lr_schedule: Learning rate schedule (could be constant)\n    :param net_arch: The specification of the policy and value networks.\n    :param activation_fn: Activation function\n    :param ortho_init: Whether to use or not orthogonal initialization\n    :param use_sde: Whether to use State Dependent Exploration or not\n    :param log_std_init: Initial value for the log standard deviation\n    :param full_std: Whether to use (n_features x n_actions) parameters\n        for the std instead of only (n_features,) when using gSDE\n    :param sde_net_arch: Network architecture for extracting features\n        when using gSDE. If None, the latent features from the policy will be used.\n        Pass an empty list to use the states as features.\n    :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure\n        a positive standard deviation (cf paper). It allows to keep variance\n        above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.\n    :param squash_output: Whether to squash the output using a tanh function,\n        this allows to ensure boundaries when using gSDE.\n    :param features_extractor_class: Features extractor to use.\n    :param features_extractor_kwargs: Keyword arguments\n        to pass to the features extractor.\n    :param normalize_images: Whether to normalize images or not,\n         dividing by 255.0 (True by default)\n    :param optimizer_class: The optimizer to use,\n        ``th.optim.Adam`` by default\n    :param optimizer_kwargs: Additional keyword arguments,\n        excluding the learning rate, to pass to the optimizer\n    ",
-        "__init__": "<function ActorCriticPolicy.__init__ at 0x7f2e52529820>",
-        "_get_constructor_parameters": "<function ActorCriticPolicy._get_constructor_parameters at 0x7f2e525298b0>",
-        "reset_noise": "<function ActorCriticPolicy.reset_noise at 0x7f2e52529940>",
-        "_build_mlp_extractor": "<function ActorCriticPolicy._build_mlp_extractor at 0x7f2e525299d0>",
-        "_build": "<function ActorCriticPolicy._build at 0x7f2e52529a60>",
-        "forward": "<function ActorCriticPolicy.forward at 0x7f2e52529af0>",
-        "_get_action_dist_from_latent": "<function ActorCriticPolicy._get_action_dist_from_latent at 0x7f2e52529b80>",
-        "_predict": "<function ActorCriticPolicy._predict at 0x7f2e52529c10>",
-        "evaluate_actions": "<function ActorCriticPolicy.evaluate_actions at 0x7f2e52529ca0>",
-        "get_distribution": "<function ActorCriticPolicy.get_distribution at 0x7f2e52529d30>",
-        "predict_values": "<function ActorCriticPolicy.predict_values at 0x7f2e52529dc0>",
+        "__doc__": "\n    Policy class for actor-critic algorithms (has both policy and value prediction).\n    Used by A2C, PPO and the likes.\n\n    :param observation_space: Observation space\n    :param action_space: Action space\n    :param lr_schedule: Learning rate schedule (could be constant)\n    :param net_arch: The specification of the policy and value networks.\n    :param activation_fn: Activation function\n    :param ortho_init: Whether to use or not orthogonal initialization\n    :param use_sde: Whether to use State Dependent Exploration or not\n    :param log_std_init: Initial value for the log standard deviation\n    :param full_std: Whether to use (n_features x n_actions) parameters\n        for the std instead of only (n_features,) when using gSDE\n    :param use_expln: Use ``expln()`` function instead of ``exp()`` to ensure\n        a positive standard deviation (cf paper). It allows to keep variance\n        above zero and prevent it from growing too fast. In practice, ``exp()`` is usually enough.\n    :param squash_output: Whether to squash the output using a tanh function,\n        this allows to ensure boundaries when using gSDE.\n    :param features_extractor_class: Features extractor to use.\n    :param features_extractor_kwargs: Keyword arguments\n        to pass to the features extractor.\n    :param share_features_extractor: If True, the features extractor is shared between the policy and value networks.\n    :param normalize_images: Whether to normalize images or not,\n         dividing by 255.0 (True by default)\n    :param optimizer_class: The optimizer to use,\n        ``th.optim.Adam`` by default\n    :param optimizer_kwargs: Additional keyword arguments,\n        excluding the learning rate, to pass to the optimizer\n    ",
+        "__init__": "<function ActorCriticPolicy.__init__ at 0x7f8a62b2bee0>",
+        "_get_constructor_parameters": "<function ActorCriticPolicy._get_constructor_parameters at 0x7f8a62b2bf70>",
+        "reset_noise": "<function ActorCriticPolicy.reset_noise at 0x7f8a62ab0040>",
+        "_build_mlp_extractor": "<function ActorCriticPolicy._build_mlp_extractor at 0x7f8a62ab00d0>",
+        "_build": "<function ActorCriticPolicy._build at 0x7f8a62ab0160>",
+        "forward": "<function ActorCriticPolicy.forward at 0x7f8a62ab01f0>",
+        "extract_features": "<function ActorCriticPolicy.extract_features at 0x7f8a62ab0280>",
+        "_get_action_dist_from_latent": "<function ActorCriticPolicy._get_action_dist_from_latent at 0x7f8a62ab0310>",
+        "_predict": "<function ActorCriticPolicy._predict at 0x7f8a62ab03a0>",
+        "evaluate_actions": "<function ActorCriticPolicy.evaluate_actions at 0x7f8a62ab0430>",
+        "get_distribution": "<function ActorCriticPolicy.get_distribution at 0x7f8a62ab04c0>",
+        "predict_values": "<function ActorCriticPolicy.predict_values at 0x7f8a62ab0550>",
         "__abstractmethods__": "frozenset()",
-        "_abc_impl": "<_abc_data object at 0x7f2e5251ed50>"
+        "_abc_impl": "<_abc_data object at 0x7f8a62b26e10>"
     },
     "verbose": 1,
     "policy_kwargs": {},
-    "observation_space": {
-        ":type:": "<class 'gym.spaces.box.Box'>",
-        ":serialized:": "gAWVbQEAAAAAAACMDmd5bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMBl9zaGFwZZRLA4WUjANsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWDAAAAAAAAAAAAIC/AACAvwAAAMGUaApLA4WUjAFDlHSUUpSMBGhpZ2iUaBIolgwAAAAAAAAAAACAPwAAgD8AAABBlGgKSwOFlGgVdJRSlIwNYm91bmRlZF9iZWxvd5RoEiiWAwAAAAAAAAABAQGUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLA4WUaBV0lFKUjA1ib3VuZGVkX2Fib3ZllGgSKJYDAAAAAAAAAAEBAZRoIUsDhZRoFXSUUpSMCl9ucF9yYW5kb22UTnViLg==",
-        "dtype": "float32",
-        "_shape": [
-            3
-        ],
-        "low": "[-1. -1. -8.]",
-        "high": "[1. 1. 8.]",
-        "bounded_below": "[ True  True  True]",
-        "bounded_above": "[ True  True  True]",
-        "_np_random": null
-    },
-    "action_space": {
-        ":type:": "<class 'gym.spaces.box.Box'>",
-        ":serialized:": "gAWV4QsAAAAAAACMDmd5bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMBl9zaGFwZZRLAYWUjANsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWBAAAAAAAAAAAAADAlGgKSwGFlIwBQ5R0lFKUjARoaWdolGgSKJYEAAAAAAAAAAAAAECUaApLAYWUaBV0lFKUjA1ib3VuZGVkX2JlbG93lGgSKJYBAAAAAAAAAAGUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLAYWUaBV0lFKUjA1ib3VuZGVkX2Fib3ZllGgSKJYBAAAAAAAAAAGUaCFLAYWUaBV0lFKUjApfbnBfcmFuZG9tlIwUbnVtcHkucmFuZG9tLl9waWNrbGWUjBJfX3JhbmRvbXN0YXRlX2N0b3KUk5SMB01UMTk5MzeUhZRSlH2UKIwNYml0X2dlbmVyYXRvcpRoMIwFc3RhdGWUfZQojANrZXmUaBIolsAJAAAAAAAAAAAAgFPCs5yHA7WTcuyrW+jmsvLPtoHa1QbvYaExTaBrtczZE+YYn3SceS/IpRuAxHkBSZ4PQ+Rd4BiTkhNVzNRKKWTSCJW+NNCeRHJ6A/Ctvwpzm3s+6WCxBtp1A0ULbn3WFmrrBDRmg7fz9aUvae9CY0O1XPfCd1LMRkQ3LQiJbtCOrnf7GLaAT3ue+U8y7BLYuY5ehakZyq49di1nK0KAnsXuxx/1IgFdS88kD8wZUmREV5hwnQr1Ehe16VreO8T/Qc86sV+2h90z2FiJqqLNf3s/yZS3bA9DHzGZfRtgkKu3Bx0ZTN7I+466APXPqRreJf4gBqUW/NB248FO9cpD0wFaed9QV63NqpiFOs/RYeEwD8e32dZFRfi9SK5aLvuApJMu2LOfZweypHIkyPjeY5W+tsm2bdPmVoCAOiyi73cf5k0LQeJNWqZU/wuy/f8myghZ9qrjf+2JSJMaB9VNMXWmxuq4Dq0fkhzJr1ML7SgftfSG29O5koFUMozQL58gAzyX96ZMcpWbZ+3/zlaVhGln5egXC8MtIK6xIFCvh/vD/F1jLgYIp14MM597MuPmTpa+OaKek7bql9Cp8/0skhg5QSCvCaijm8wenxrfqLyRxDPCpS+L3isJC5LrjFgWnsdxQXVrJK8uaobJcTIJ5NrYYfA2l5gH27iPI9EqhzFtZJXiE4vXpH18f3kouYV9RowPzOtmYsbmstR/Mx/VY7E0XBmnMidL4dYTUXgxaDbFxWy3y6miL2yw0I2O09vPWV7LqbwMbthlU26lrLfnJDz88B+7y4pFCwvsHgCsMWq6pvroAF4Ms+++JnhzrL0GLrtfJ7667p42Vg78GirmKIRMFH0p6aLRPV4V/fclE3PLBj0InU315M5v7fDFj/IreJeFUhIAy5/BlvLdgwIfBMBWvyjhqGRBo2KmdiDAO9BPhdi6oGmZUCxTDjUyMd8rjeRdntTE+L9sHQUSvUfpRjKcSIjZXcjsMMSwwU0QzIXewD4nZg8EAZ72iHnChWveW7cB5EQRFE+YPvvyNAw4OvORF+DjDtmLUz6jNz0JbkuKbhIQEEVbpDRkoeIGA4HATymZeE4TX/hvhOxJfX5liXD9/Uon8OjJ/hhIrDNUYR2NOV2R1L+VuK/JA9o0izGPiCC9vi7UK1u4re8tvPrBInC6BH2DpMvWSRepdH0frFxGsH/kRB/S77USeMs38a+yorDaa9Wsb9WnuQg10vl087jhvwSZx77fGq52NV1h/UnZAjnqJOjCPBgQrY0wsSeSW6SOSdItAOupiWkVvSzHz+gopd/3FUUPbjdik7Vd6O6ycGOFwp4+wyZqq1MKHww5exJND6De8NB4fGBzsvKoT8O1fAC3Y2Z+3mLYwkCJXULx2zNSO90bQi0L0BhTF1AidNtqRkak72V+V1kos6m4F+kyZiOiwxfcHKWHmpQe3A8HpX6nacx0Zso0WyF/W/sOLo//2g0YD9koIjgKvw//f9Em4CvxNBUMnJYTVQJGysF4QMShnM6byaMNJhRVALf+X0+SgLJkEwLfKdBbKxjA035OEc2YKmUpvooVUUt+9U8d7cKRgNYKKPwCHyYfLJSQt4ZEdThgjeywDxgsGzPax5SklOLroSYn39feofatZDzJXevfPTHyi1ZLwpe6Hwkbqz1FuOnHiShPedbEA9b/HCtBytSZrgUwlwHpQlAiBTSxpN7TFzVZrrhRd6N8W+FeE9SAGCHwXchFR4SCGThPyRGO/XWkMPJ42BLUOmMGepDQgWH646tjoJSE3EXcA5iDS+Nq6Oh521oC2UPAnvxIj1QbVh8IbxlBytRTTjc0c14E9cyhIPlgIoHriuEFSMXSEzOGL1MmL6UCbiXfsRg9Z6OwWCCl3VeGg5bEZ1kjJkvs08k7wtPk4ATAjaTL3QoY2gf106zFbJtL4D5gmLMJ9OuzE2Fn5uaAqqpjXIqqEXxS9jtpsRU9VTHCg68RwXQVIUhuVJgHq8fOigBMrW7Am5+jjo/GNNlcFcp813dFiXy4qHhjGSEjNXp3ln03NZkOgqXQ8SalJlPOvyrAS9wW9EtjQKhcrBSWSsQ8C3o/Mc/sR/CMRB85ZIojR/tiCKtOutxQMIusIOnYHK8g6kPpTQ8J/PfJ8pa3GEoYoA67axQTXsysd2Y6ZDwpz1HkAeISVK2AlCcuQssrQv8dVLAcins/2kjRFp5Vp82HSX9j6Ci9GH5mkdyqV84vWsdwRz4JNXHZoHVZKnrxSdA1HUwRUI/5oWiqnGk9KyiS4Mv9dQIVluJ1+/pAHYEdG9YgLHiNE2zA7aIQbqqlGX6jH1CrHSPL9mnlHdPiKjozwRXu02UQuzlGJn+/PUkU6cPYLLeLc8e7S2qfCZxbdpHioand7wYKqb5bMb8dA3Dwvm6P6iJXSogJ+Q+0z43li8ydYJqZNZlSjsljr/2c5UU33vMhLDLEXIHT8WzrsS45TsKNbhYfTYx6Ds/8W5yOtiOPWc0+fRlAjbQC++FKo5UaMl2eRCxI4U5/heX2HxJGNk50rICjuFsG/8Q+NUuJgS+y3FMhe+sm1e2MdC+ldkBqEn8oRxajECbP4Wizz1tfJliW/1A5fdGTfUxM3HV72bJgnqswmoAumAx6d36KfuZwEEp0/wcrdo+8/unJ5f2mYeqCrOcaxDJrs6SxW1zVaH/YTZl+RNA0NjPLgaqnlveaes/MkpzsVEQDtvKGFrG1cnmGjZVi2azrSDGQ0Y423nEksDC5awYcOJmVYbeA4DkMSNfj+7Dx2SzH/PVPuXLX9aw9K9QF4Ml48zsSrwsVjIa8+gIdffs2pf2wCKcPtFez6vOT0UDuFHknJjMDg0fI3DnyC7jJqO8V4XpmPyarTp3JJRGhmqTHhpZInn70JMfS+RFry5+rLSOM0T+KWV8fYjs9eyCiZijlR4AiADooXm9G8JIzZCLZX2Dty83iyz7gQzSxYO7ULuTT1stvGuJwbBP4LMhLXkbxdhAmBSDiYNOnc3O+yFsO6Ps9UOQD8S4Pbr8hZ4mFjbicpO635SwpmHINYDeuewln3/GHz69LpCjmpnKPeF9ZxXcq6MR4kJUV2j/dQzqjLniNaQmrMkULdI7W1sMXRFcsz9xs1GVwVqmtMVws8HtvXMYNmosCrrgAFX2ghPz7dXCV6vML5YhfNbDAzzG6MHffrslrhMav3vtlt8Fnld4VaH6IhMkowayT1lSVvfvlKHCWwtKaTcOZrR5LZGalJOpFbVIFUOAo+LnY/25bmc3KloyLzgiTudjPsXEGPNPBIvE/5cMEvU4Lrs0N3tCke4abYDXF9f14QrwLlGgHjAJ1NJSJiIeUUpQoSwNoC05OTkr/////Sv////9LAHSUYk1wAoWUaBV0lFKUjANwb3OUTXACdYwJaGFzX2dhdXNzlEsAjAVnYXVzc5RHAAAAAAAAAAB1YnViLg==",
-        "dtype": "float32",
-        "_shape": [
-            1
-        ],
-        "low": "[-2.]",
-        "high": "[2.]",
-        "bounded_below": "[ True]",
-        "bounded_above": "[ True]",
-        "_np_random": "RandomState(MT19937)"
-    },
-    "n_envs": 4,
     "num_timesteps": 102400,
     "_total_timesteps": 100000,
     "_num_timesteps_at_start": 0,
     "seed": 0,
     "action_noise": null,
-    "start_time": 1659105256.5056021,
+    "start_time": 1694771152245526076,
     "learning_rate": {
         ":type:": "<class 'function'>",
-        ":serialized:": "gAWVhQIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwNX2J1aWx0aW5fdHlwZZSTlIwKTGFtYmRhVHlwZZSFlFKUKGgCjAhDb2RlVHlwZZSFlFKUKEsBSwBLAEsBSwFLE0MEiABTAJROhZQpjAFflIWUjFUvaG9tZS9tYXhpbWlsaWFuLy5sb2NhbC9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLgEMCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgNdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoH32UfZQoaBdoDowMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBiMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP1BiTdLxqfyFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
-    },
-    "tensorboard_log": "runs/Pendulum-v1__ppo__8__1659105255/Pendulum-v1",
-    "lr_schedule": {
-        ":type:": "<class 'function'>",
-        ":serialized:": "gAWVhQIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwNX2J1aWx0aW5fdHlwZZSTlIwKTGFtYmRhVHlwZZSFlFKUKGgCjAhDb2RlVHlwZZSFlFKUKEsBSwBLAEsBSwFLE0MEiABTAJROhZQpjAFflIWUjFUvaG9tZS9tYXhpbWlsaWFuLy5sb2NhbC9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLgEMCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgNdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoH32UfZQoaBdoDowMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBiMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP1BiTdLxqfyFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
+        ":serialized:": "gAWVlwIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAUsTQwSIAFMAlE6FlCmMAV+UhZSMZS9ob21lL21heGltaWxpYW4vcmwtYmFzZWxpbmVzMy16b28vdmVudi9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLg0MCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgMdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHn2UfZQoaBZoDYwMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBeMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP1BiTdLxqfyFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
     },
+    "tensorboard_log": null,
     "_last_obs": null,
     "_last_episode_starts": {
         ":type:": "<class 'numpy.ndarray'>",
@@ -72,15 +42,47 @@
     "use_sde": true,
     "sde_sample_freq": 4,
     "_current_progress_remaining": -0.02400000000000002,
+    "_stats_window_size": 100,
     "ep_info_buffer": {
         ":type:": "<class 'collections.deque'>",
-        ":serialized:": "gAWVHRAAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKUKH2UKIwBcpSMFW51bXB5LmNvcmUubXVsdGlhcnJheZSMBnNjYWxhcpSTlIwFbnVtcHmUjAVkdHlwZZSTlIwCZjiUiYiHlFKUKEsDjAE8lE5OTkr/////Sv////9LAHSUYkMIxRwEHa1accCUhpRSlIwBbJRLyIwBdJRHQEvB6ciGFi91fZQoaAZoCWgPQwhcPpKSHgYDwJSGlFKUaBVLyGgWR0BLwbr1M/QjdX2UKGgGaAloD0MIN3Fyv0PTcMCUhpRSlGgVS8hoFkdAS8Gkep4r0HV9lChoBmgJaA9DCMoXtJAA03DAlIaUUpRoFUvIaBZHQEvBjxTbWVh1fZQoaAZoCWgPQwio34Wt2SdxwJSGlFKUaBVLyGgWR0BL1FAeJYT1dX2UKGgGaAloD0MIrDyBsFOsA8CUhpRSlGgVS8hoFkdAS9QhB7eEZnV9lChoBmgJaA9DCCodrP9z22DAlIaUUpRoFUvIaBZHQEvUCvovBad1fZQoaAZoCWgPQwjvVpbo7Il6wJSGlFKUaBVLyGgWR0BL0/V7Qb++dX2UKGgGaAloD0MI9l0R/K9lecCUhpRSlGgVS8hoFkdAS+bQzDXOGHV9lChoBmgJaA9DCFLTLqYZj3jAlIaUUpRoFUvIaBZHQEvmoegctGx1fZQoaAZoCWgPQwiE9X8Oc4hgwJSGlFKUaBVLyGgWR0BL5ow/PgNxdX2UKGgGaAloD0MIdELooOuVgcCUhpRSlGgVS8hoFkdAS+Z3HJcPfHV9lChoBmgJaA9DCF95kJ4iwnDAlIaUUpRoFUvIaBZHQEv5QSi/O+t1fZQoaAZoCWgPQwgKuVLPgnBgwJSGlFKUaBVLyGgWR0BL+RJNCZ4OdX2UKGgGaAloD0MIfy4aMt69eMCUhpRSlGgVS8hoFkdAS/j8WKuSwHV9lChoBmgJaA9DCHu8kA7Pd3DAlIaUUpRoFUvIaBZHQEv45wwTM7l1fZQoaAZoCWgPQwiX/brTXRV4wJSGlFKUaBVLyGgWR0BNAfg75mAcdX2UKGgGaAloD0MImFEst7Qa+b+UhpRSlGgVS8hoFkdATQHNX5nDi3V9lChoBmgJaA9DCMjT8gPXJ3DAlIaUUpRoFUvIaBZHQE0Budf9gnd1fZQoaAZoCWgPQwgAPKJCdTP/v5SGlFKUaBVLyGgWR0BNAadlNDc/dX2UKGgGaAloD0MINbVsrS+QYMCUhpRSlGgVS8hoFkdATR+GfwqiGnV9lChoBmgJaA9DCNREn48y72DAlIaUUpRoFUvIaBZHQE0fWy1NQCV1fZQoaAZoCWgPQwjusfShC+hwwJSGlFKUaBVLyGgWR0BNH0gKWszVdX2UKGgGaAloD0MIqmG/Jxanf8CUhpRSlGgVS8hoFkdATR82DQJHAnV9lChoBmgJaA9DCKlqgqg7GnrAlIaUUpRoFUvIaBZHQE09SSeRPoF1fZQoaAZoCWgPQwgJNxlVRnh6wJSGlFKUaBVLyGgWR0BNPR33YcvNdX2UKGgGaAloD0MIb38uGrKYb8CUhpRSlGgVS8hoFkdATT0KPXCj13V9lChoBmgJaA9DCFneVQ+YFmDAlIaUUpRoFUvIaBZHQE0892X9itt1fZQoaAZoCWgPQwjbFI+LatH/v5SGlFKUaBVLyGgWR0BNWxNATqSpdX2UKGgGaAloD0MI24toOyY1dsCUhpRSlGgVS8hoFkdATVrnX/YJ3XV9lChoBmgJaA9DCEH0pEwq2HDAlIaUUpRoFUvIaBZHQE1a0+C9RJp1fZQoaAZoCWgPQwg7b2Ozo/RgwJSGlFKUaBVLyGgWR0BNWsIu5BkadX2UKGgGaAloD0MI9PqT+JwhccCUhpRSlGgVS8hoFkdATXjYXfqHGnV9lChoBmgJaA9DCDV+4ZWkiWDAlIaUUpRoFUvIaBZHQE14rJ8v25B1fZQoaAZoCWgPQwhsPq4NFedewJSGlFKUaBVLyGgWR0BNeJj+aScLdX2UKGgGaAloD0MIP/89eG33YMCUhpRSlGgVS8hoFkdATXiGUOd5IHV9lChoBmgJaA9DCJXXSuguZWDAlIaUUpRoFUvIaBZHQE7P49ovi991fZQoaAZoCWgPQwgCmggbnmZvwJSGlFKUaBVLyGgWR0BOz7UXpGF0dX2UKGgGaAloD0MIJJur5nlPdsCUhpRSlGgVS8hoFkdATs+fI0ZWJnV9lChoBmgJaA9DCNV6v9FOKnzAlIaUUpRoFUvIaBZHQE7PiZv1lGx1fZQoaAZoCWgPQwg7GLFPgMFgwJSGlFKUaBVLyGgWR0BO4s+3Ytg8dX2UKGgGaAloD0MIjWDj+vfOcMCUhpRSlGgVS8hoFkdATuKg7HQyAXV9lChoBmgJaA9DCLHAV3TrFQDAlIaUUpRoFUvIaBZHQE7iipNsWO91fZQoaAZoCWgPQwgm4NdIkgN5wJSGlFKUaBVLyGgWR0BO4nUc4o7WdX2UKGgGaAloD0MIOQoQBTMm9L+UhpRSlGgVS8hoFkdATvX2GqPwNXV9lChoBmgJaA9DCGEcXDrmvAjAlIaUUpRoFUvIaBZHQE71x1gYxcp1fZQoaAZoCWgPQwj7zcR0IcdgwJSGlFKUaBVLyGgWR0BO9bDVH4GmdX2UKGgGaAloD0MIP/7Sor71YMCUhpRSlGgVS8hoFkdATvWbd8Aq/nV9lChoBmgJaA9DCDiHa7UHyG7AlIaUUpRoFUvIaBZHQE8IeOn2qT91fZQoaAZoCWgPQwiTcCGP4JpgwJSGlFKUaBVLyGgWR0BPCEo4MnZ1dX2UKGgGaAloD0MIM1Naf0ssf8CUhpRSlGgVS8hoFkdATwgz1schknV9lChoBmgJaA9DCIgq/BnebAbAlIaUUpRoFUvIaBZHQE8IHnlnyup1fZQoaAZoCWgPQwioc0UpIf1fwJSGlFKUaBVLyGgWR0BPGvSc9W6tdX2UKGgGaAloD0MIXHaIf9jyAsCUhpRSlGgVS8hoFkdATxrF6zE74nV9lChoBmgJaA9DCEm6ZvIN1XfAlIaUUpRoFUvIaBZHQE8ar6LwWnF1fZQoaAZoCWgPQwiOOjquxi18wJSGlFKUaBVLyGgWR0BPGposZpBYdX2UKGgGaAloD0MIUirhCb1+97+UhpRSlGgVS8hoFkdAT/PvWpZOi3V9lChoBmgJaA9DCCVBuAKKMGDAlIaUUpRoFUvIaBZHQE/zwBHTZxt1fZQoaAZoCWgPQwhu+x7118sJwJSGlFKUaBVLyGgWR0BP86kAPuohdX2UKGgGaAloD0MIeqhtwyiIAsCUhpRSlGgVS8hoFkdAT/OTV2A5JnV9lChoBmgJaA9DCOZbH9abnmDAlIaUUpRoFUvIaBZHQFADattALRd1fZQoaAZoCWgPQwjdQlciUMBgwJSGlFKUaBVLyGgWR0BQA1NlAeJYdX2UKGgGaAloD0MIptO6DWpPcMCUhpRSlGgVS8hoFkdAUANIVdonKHV9lChoBmgJaA9DCOjc7XrpBWHAlIaUUpRoFUvIaBZHQFADPatcOb11fZQoaAZoCWgPQwiEKjV7oOBgwJSGlFKUaBVLyGgWR0BQDKu0TlDGdX2UKGgGaAloD0MI1uWUgJgkAsCUhpRSlGgVS8hoFkdAUAyUNayKN3V9lChoBmgJaA9DCOVFJuBXvmDAlIaUUpRoFUvIaBZHQFAMiXY150N1fZQoaAZoCWgPQwjpSC7/IelgwJSGlFKUaBVLyGgWR0BQDH8CPp6hdX2UKGgGaAloD0MI5DEDlXHHYMCUhpRSlGgVS8hoFkdAUBY1Gb1AaHV9lChoBmgJaA9DCOyi6IGP0WDAlIaUUpRoFUvIaBZHQFAWHaN+9al1fZQoaAZoCWgPQwjVPh2PmQFgwJSGlFKUaBVLyGgWR0BQFhLbpNbkdX2UKGgGaAloD0MI51HxfwcYcMCUhpRSlGgVS8hoFkdAUBYILPUrkXV9lChoBmgJaA9DCP27PnOWKXfAlIaUUpRoFUvIaBZHQFAfbBXS0Bx1fZQoaAZoCWgPQwgB/FOqRIVwwJSGlFKUaBVLyGgWR0BQH1S88La3dX2UKGgGaAloD0MIZ+22C82McMCUhpRSlGgVS8hoFkdAUB9J+UhV2nV9lChoBmgJaA9DCOY9zjThIGDAlIaUUpRoFUvIaBZHQFAfP3BYV7B1fZQoaAZoCWgPQwinBMQkXAgQwJSGlFKUaBVLyGgWR0BQjlUdaMaTdX2UKGgGaAloD0MIGm7A54ftXcCUhpRSlGgVS8hoFkdAUI49r433pXV9lChoBmgJaA9DCBGMg0tHKGDAlIaUUpRoFUvIaBZHQFCOMrmQr+Z1fZQoaAZoCWgPQwgMQKN0abRgwJSGlFKUaBVLyGgWR0BQjigK4QSSdX2UKGgGaAloD0MILBGo/kH8bsCUhpRSlGgVS8hoFkdAUJef5DZ13nV9lChoBmgJaA9DCNSbUfMVsHzAlIaUUpRoFUvIaBZHQFCXiF0xM391fZQoaAZoCWgPQwjrrBbY45RuwJSGlFKUaBVLyGgWR0BQl30PH1e0dX2UKGgGaAloD0MIWdx/ZLpKb8CUhpRSlGgVS8hoFkdAUJdyimEXcnV9lChoBmgJaA9DCN/eNejLSG7AlIaUUpRoFUvIaBZHQFCg1yeZof11fZQoaAZoCWgPQwiq7/yiRMl3wJSGlFKUaBVLyGgWR0BQoL9/BnBddX2UKGgGaAloD0MI7xtfe+Z2YMCUhpRSlGgVS8hoFkdAUKC0pmVZ93V9lChoBmgJaA9DCMN/uoGC9WDAlIaUUpRoFUvIaBZHQFCgqhDgIhR1fZQoaAZoCWgPQwj7dhIR/hRhwJSGlFKUaBVLyGgWR0BQwm1+iJwbdX2UKGgGaAloD0MIF4IclHCMcMCUhpRSlGgVS8hoFkdAUMJV3ljmS3V9lChoBmgJaA9DCCUgJuFCnvO/lIaUUpRoFUvIaBZHQFDCSrYGt6p1fZQoaAZoCWgPQwg25QrvcspewJSGlFKUaBVLyGgWR0BQwkADJU5udX2UKGgGaAloD0MInkFD/4Tvc8CUhpRSlGgVS8hoFkdAUMvUlRgqmXV9lChoBmgJaA9DCLPsSWAzOXfAlIaUUpRoFUvIaBZHQFDLvUjLSu11fZQoaAZoCWgPQwjjxi3mp296wJSGlFKUaBVLyGgWR0BQy7JCBwuNdX2UKGgGaAloD0MIBTHQta+8ccCUhpRSlGgVS8hoFkdAUMunl4keIXV9lChoBmgJaA9DCNhHp668SHbAlIaUUpRoFUvIaBZHQFDVE/jbSJF1fZQoaAZoCWgPQwh2qRH6mQ1xwJSGlFKUaBVLyGgWR0BQ1Pxc3VCpdX2UKGgGaAloD0MIob5lThc+ccCUhpRSlGgVS8hoFkdAUNTxSYPXkHV9lChoBmgJaA9DCOmdCrjns2DAlIaUUpRoFUvIaBZHQFDU5ooNNJx1ZS4="
+        ":serialized:": "gAWV4AsAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKUKH2UKIwBcpRHwHDGmeYlY2eMAWyUS8iMAXSUR0BGSomw7kn1dX2UKGgGR8AQyaz/p+tsaAdLyGgIR0BGSnVXmvGIdX2UKGgGR8AMLJEH+qBFaAdLyGgIR0BGSmLUCq6wdX2UKGgGR8B5j9ekYXO4aAdLyGgIR0BGSkcbR4QjdX2UKGgGR8APwaxX4j8laAdLyGgIR0BGWjh99c8ldX2UKGgGR8BhPPPE87p3aAdLyGgIR0BGWiPyTY/WdX2UKGgGR8ASjGuLaVUuaAdLyGgIR0BGWhFmWdEtdX2UKGgGR8BwfNFkQPI5aAdLyGgIR0BGWfWlMyrQdX2UKGgGR8CFGldkauOkaAdLyGgIR0BGad+5OJtSdX2UKGgGR8CA5kchC+lCaAdLyGgIR0BGacs+V1OkdX2UKGgGR8BwTm4YrJ8waAdLyGgIR0BGabi6xxDLdX2UKGgGR8CJGHyp71IzaAdLyGgIR0BGaZz5oGpudX2UKGgGR8CA/JUZvUBoaAdLyGgIR0BGeYP5HmRvdX2UKGgGR8Bw3S5VfeDWaAdLyGgIR0BGeW9lEqlQdX2UKGgGR8CIlUYk3S8baAdLyGgIR0BGeVzySV4YdX2UKGgGR8CA40XBxgiNaAdLyGgIR0BGeUExIre7dX2UKGgGR8B5mDK7qY7aaAdLyGgIR0BHNQ6IWP92dX2UKGgGR8CFwBiXpnpTaAdLyGgIR0BHNPvSc9W7dX2UKGgGR8Bg8oYtQKrraAdLyGgIR0BHNOuaF23bdX2UKGgGR8CFQXJlJ6IFaAdLyGgIR0BHNNAkcCHRdX2UKGgGR8Bw8faVUuL8aAdLyGgIR0BHROMl1KXfdX2UKGgGR8CEsaKYzBRAaAdLyGgIR0BHRM7+1jRVdX2UKGgGR8BxDxDRc/t6aAdLyGgIR0BHRLyUcGTtdX2UKGgGR8BxOEgdOqNqaAdLyGgIR0BHRKDbrTpgdX2UKGgGR8B5Fl0cOskqaAdLyGgIR0BHVLsrupjudX2UKGgGR8APgIhQm/nGaAdLyGgIR0BHVKd6LOzIdX2UKGgGR8CIuSpH7P6baAdLyGgIR0BHVJUYKpkxdX2UKGgGR8BwilVsDW9UaAdLyGgIR0BHVHmJWNm2dX2UKGgGR8B55/m8ujASaAdLyGgIR0BHZJ9RaX8gdX2UKGgGR8Bwxb7MxGlRaAdLyGgIR0BHZIuGsV+JdX2UKGgGR8B5YTz9S/CZaAdLyGgIR0BHZHkkrwvydX2UKGgGR8B4OhPAO8TSaAdLyGgIR0BHZF18stkGdX2UKGgGR8CBBlhFVktmaAdLyGgIR0BHdIj4YaYNdX2UKGgGR8B/qcOLBKtgaAdLyGgIR0BHdHTRYzSDdX2UKGgGR8CAtnU4JeE7aAdLyGgIR0BHdGKQ7tAtdX2UKGgGR8CJs/XJYDDCaAdLyGgIR0BHdEidJ8OTdX2UKGgGR8CFAeZAIIGAaAdLyGgIR0BILC/wiJO4dX2UKGgGR8ASqW6bvw3HaAdLyGgIR0BILB1cMVk+dX2UKGgGR8CFEAiFj/dZaAdLyGgIR0BILAsK9f1IdX2UKGgGR8AO3HvMKTjeaAdLyGgIR0BIK++Eh7mddX2UKGgGR8Bwqim/FirlaAdLyGgIR0BIO9pAUtZndX2UKGgGR8BxBMIIF/x2aAdLyGgIR0BIO8YIjW07dX2UKGgGR8ARDbi6xxDLaAdLyGgIR0BIO7OeJ53UdX2UKGgGR8CAKk619fCzaAdLyGgIR0BIO5flZHNHdX2UKGgGR8AXVB4Uvf0maAdLyGgIR0BIS38XN1QqdX2UKGgGR8BheVuYQarFaAdLyGgIR0BIS2rn1WbPdX2UKGgGR8Bw+zhuO0b+aAdLyGgIR0BIS1h9b5dodX2UKGgGR8B27lRUFSsKaAdLyGgIR0BISzzND+irdX2UKGgGR8BhnkF6iTMaaAdLyGgIR0BIW1QIldC3dX2UKGgGR8ARmVgQYk3TaAdLyGgIR0BIW0Gmk30gdX2UKGgGR8Bxb7N0NjLCaAdLyGgIR0BIWy9ugpSadX2UKGgGR8B88+jafzz3aAdLyGgIR0BIWxQBPsRhdX2UKGgGR8B5zzgn+hoNaAdLyGgIR0BIa0DEFW4mdX2UKGgGR8B3x5GH58BuaAdLyGgIR0BIay1/lQuVdX2UKGgGR8AMGXw9aEBbaAdLyGgIR0BIaxsMy8BddX2UKGgGR8Bw9o9Mbm2caAdLyGgIR0BIav9tMwlCdX2UKGgGR8B2t53GGVRlaAdLyGgIR0BJKes5n13/dX2UKGgGR8Bw6IKpkwvhaAdLyGgIR0BJKdkBjnV5dX2UKGgGR8CIvjS1maphaAdLyGgIR0BJKcbaRISUdX2UKGgGR8B3XAnWrfcfaAdLyGgIR0BJKatLcsUZdX2UKGgGR8BxNZV7x/d7aAdLyGgIR0BJOe54GD+SdX2UKGgGR8CA1fZZjhDPaAdLyGgIR0BJOdvsJIDpdX2UKGgGR8BhXzaIvalDaAdLyGgIR0BJOcma6STydX2UKGgGR8CLm4ZqmCRPaAdLyGgIR0BJOa4MF2V3dX2UKGgGR8BhpbdFfAsTaAdLyGgIR0BJSd8Z1mrbdX2UKGgGR8CAx+Ik7fYSaAdLyGgIR0BJSctGus90dX2UKGgGR8Bw69J17pmmaAdLyGgIR0BJSbj1f3N+dX2UKGgGR8CGYiiXY150aAdLyGgIR0BJSZ1FH8TBdX2UKGgGR8CFImAkLQXzaAdLyGgIR0BJWd6C17Y1dX2UKGgGR7/0Q5/9YOlPaAdLyGgIR0BJWcriEQGwdX2UKGgGR8B6bUsoUi6haAdLyGgIR0BJWbiIcinpdX2UKGgGR8BhVVoQFs55aAdLyGgIR0BJWZzgdfb9dX2UKGgGR8BxGVpVS4vwaAdLyGgIR0BJakbo8p1BdX2UKGgGR8B5L/ueBg/kaAdLyGgIR0BJajSPU8V6dX2UKGgGR8B5a3syBTXKaAdLyGgIR0BJaiJO32EkdX2UKGgGR8BgyZoPCl7/aAdLyGgIR0BJagggX/HYdX2UKGgGR8BhzBksjFAFaAdLyGgIR0BKJr2pQ1rJdX2UKGgGR8AOSi9IwudxaAdLyGgIR0BKJqxs2vSudX2UKGgGR8CFY12wmmcfaAdLyGgIR0BKJpo0ygwodX2UKGgGR8B5k8l+mWMTaAdLyGgIR0BKJn6l+EytdX2UKGgGR8BxK7hisny/aAdLyGgIR0BKNpeVs1sMdX2UKGgGR8AD/huO0b97aAdLyGgIR0BKNoMKCxu9dX2UKGgGR8Bw0tmHxjJ/aAdLyGgIR0BKNnKOktVadX2UKGgGR8B5hz2lEZzgaAdLyGgIR0BKNlbNbC79dX2UKGgGR8B41pAfMfRvaAdLyGgIR0BKRmRvFWGRdX2UKGgGR8BwyCVQhwERaAdLyGgIR0BKRlAeJYT1dX2UKGgGR8BwROJUHY6GaAdLyGgIR0BKRj2zv7WNdX2UKGgGR8CAG0yJKraNaAdLyGgIR0BKRiILw4KhdX2UKGgGR8BgpeG21D0EaAdLyGgIR0BKVkz41xbTdX2UKGgGR8CFLb/qgRK6aAdLyGgIR0BKVjh1klNUdX2UKGgGR8AIEP+XJHRUaAdLyGgIR0BKViX6ZYxMdX2UKGgGR8BxHWYjSofkaAdLyGgIR0BKVgo5PuXvdX2UKGgGR8ATcO/cnE2paAdLyGgIR0BKZgiNbTttdX2UKGgGR8Bw+yTnq3VkaAdLyGgIR0BKZfSYw7DEdX2UKGgGR8CFISGIsRQKaAdLyGgIR0BKZeIuXeFddX2UKGgGR8BwuDxaxHG0aAdLyGgIR0BKZcaOxSpBdX2UKGgGR8Bwx9L7GecyaAdLyGgIR0BKdclXzUZvdX2UKGgGR8BxFe3KB/ZvaAdLyGgIR0BKdbTc6/7BdX2UKGgGR8CJEIh+OOsDaAdLyGgIR0BKdaJhvze5dX2UKGgGR8CFBG2pAD7qaAdLyGgIR0BKdYa5wwTNdWUu"
     },
     "ep_success_buffer": {
         ":type:": "<class 'collections.deque'>",
         ":serialized:": "gAWVIAAAAAAAAACMC2NvbGxlY3Rpb25zlIwFZGVxdWWUk5QpS2SGlFKULg=="
     },
     "_n_updates": 250,
+    "observation_space": {
+        ":type:": "<class 'gymnasium.spaces.box.Box'>",
+        ":serialized:": "gAWVpwEAAAAAAACMFGd5bW5hc2l1bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMDWJvdW5kZWRfYmVsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWAwAAAAAAAAABAQGUaAeMAmIxlImIh5RSlChLA4wBfJROTk5K/////0r/////SwB0lGJLA4WUjAFDlHSUUpSMDWJvdW5kZWRfYWJvdmWUaBAolgMAAAAAAAAAAQEBlGgUSwOFlGgYdJRSlIwGX3NoYXBllEsDhZSMA2xvd5RoECiWDAAAAAAAAAAAAIC/AACAvwAAAMGUaApLA4WUaBh0lFKUjARoaWdolGgQKJYMAAAAAAAAAAAAgD8AAIA/AAAAQZRoCksDhZRoGHSUUpSMCGxvd19yZXBylIwNWy0xLiAtMS4gLTguXZSMCWhpZ2hfcmVwcpSMClsxLiAxLiA4Ll2UjApfbnBfcmFuZG9tlE51Yi4=",
+        "dtype": "float32",
+        "bounded_below": "[ True  True  True]",
+        "bounded_above": "[ True  True  True]",
+        "_shape": [
+            3
+        ],
+        "low": "[-1. -1. -8.]",
+        "high": "[1. 1. 8.]",
+        "low_repr": "[-1. -1. -8.]",
+        "high_repr": "[1. 1. 8.]",
+        "_np_random": null
+    },
+    "action_space": {
+        ":type:": "<class 'gymnasium.spaces.box.Box'>",
+        ":serialized:": "gAWVRgIAAAAAAACMFGd5bW5hc2l1bS5zcGFjZXMuYm94lIwDQm94lJOUKYGUfZQojAVkdHlwZZSMBW51bXB5lGgFk5SMAmY0lImIh5RSlChLA4wBPJROTk5K/////0r/////SwB0lGKMDWJvdW5kZWRfYmVsb3eUjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWAQAAAAAAAAABlGgHjAJiMZSJiIeUUpQoSwOMAXyUTk5OSv////9K/////0sAdJRiSwGFlIwBQ5R0lFKUjA1ib3VuZGVkX2Fib3ZllGgQKJYBAAAAAAAAAAGUaBRLAYWUaBh0lFKUjAZfc2hhcGWUSwGFlIwDbG93lGgQKJYEAAAAAAAAAAAAAMCUaApLAYWUaBh0lFKUjARoaWdolGgQKJYEAAAAAAAAAAAAAECUaApLAYWUaBh0lFKUjAhsb3dfcmVwcpSMBC0yLjCUjAloaWdoX3JlcHKUjAMyLjCUjApfbnBfcmFuZG9tlIwUbnVtcHkucmFuZG9tLl9waWNrbGWUjBBfX2dlbmVyYXRvcl9jdG9ylJOUjAVQQ0c2NJRoMYwUX19iaXRfZ2VuZXJhdG9yX2N0b3KUk5SGlFKUfZQojA1iaXRfZ2VuZXJhdG9ylIwFUENHNjSUjAVzdGF0ZZR9lChoPIoQ42GVprdeWAktRZZZNLWhGowDaW5jlIoQqXN4RLwzgViCGvc629qNQXWMCmhhc191aW50MzKUSwCMCHVpbnRlZ2VylEsAdWJ1Yi4=",
+        "dtype": "float32",
+        "bounded_below": "[ True]",
+        "bounded_above": "[ True]",
+        "_shape": [
+            1
+        ],
+        "low": "[-2.]",
+        "high": "[2.]",
+        "low_repr": "-2.0",
+        "high_repr": "2.0",
+        "_np_random": "Generator(PCG64)"
+    },
+    "n_envs": 1,
     "n_steps": 1024,
     "gamma": 0.9,
     "gae_lambda": 0.95,
@@ -91,9 +93,13 @@
     "n_epochs": 10,
     "clip_range": {
         ":type:": "<class 'function'>",
-        ":serialized:": "gAWVhQIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwNX2J1aWx0aW5fdHlwZZSTlIwKTGFtYmRhVHlwZZSFlFKUKGgCjAhDb2RlVHlwZZSFlFKUKEsBSwBLAEsBSwFLE0MEiABTAJROhZQpjAFflIWUjFUvaG9tZS9tYXhpbWlsaWFuLy5sb2NhbC9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLgEMCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgNdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoH32UfZQoaBdoDowMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBiMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP8mZmZmZmZqFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
+        ":serialized:": "gAWVlwIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAUsTQwSIAFMAlE6FlCmMAV+UhZSMZS9ob21lL21heGltaWxpYW4vcmwtYmFzZWxpbmVzMy16b28vdmVudi9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLg0MCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgMdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHn2UfZQoaBZoDYwMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBeMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP8mZmZmZmZqFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
     },
     "clip_range_vf": null,
     "normalize_advantage": true,
-    "target_kl": null
+    "target_kl": null,
+    "lr_schedule": {
+        ":type:": "<class 'function'>",
+        ":serialized:": "gAWVlwIAAAAAAACMF2Nsb3VkcGlja2xlLmNsb3VkcGlja2xllIwOX21ha2VfZnVuY3Rpb26Uk5QoaACMDV9idWlsdGluX3R5cGWUk5SMCENvZGVUeXBllIWUUpQoSwFLAEsASwFLAUsTQwSIAFMAlE6FlCmMAV+UhZSMZS9ob21lL21heGltaWxpYW4vcmwtYmFzZWxpbmVzMy16b28vdmVudi9saWIvcHl0aG9uMy44L3NpdGUtcGFja2FnZXMvc3RhYmxlX2Jhc2VsaW5lczMvY29tbW9uL3V0aWxzLnB5lIwEZnVuY5RLg0MCAAGUjAN2YWyUhZQpdJRSlH2UKIwLX19wYWNrYWdlX1+UjBhzdGFibGVfYmFzZWxpbmVzMy5jb21tb26UjAhfX25hbWVfX5SMHnN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi51dGlsc5SMCF9fZmlsZV9flGgMdU5OaACMEF9tYWtlX2VtcHR5X2NlbGyUk5QpUpSFlHSUUpSMHGNsb3VkcGlja2xlLmNsb3VkcGlja2xlX2Zhc3SUjBJfZnVuY3Rpb25fc2V0c3RhdGWUk5RoHn2UfZQoaBZoDYwMX19xdWFsbmFtZV9flIwZY29uc3RhbnRfZm4uPGxvY2Fscz4uZnVuY5SMD19fYW5ub3RhdGlvbnNfX5R9lIwOX19rd2RlZmF1bHRzX1+UTowMX19kZWZhdWx0c19flE6MCl9fbW9kdWxlX1+UaBeMB19fZG9jX1+UTowLX19jbG9zdXJlX1+UaACMCl9tYWtlX2NlbGyUk5RHP1BiTdLxqfyFlFKUhZSMF19jbG91ZHBpY2tsZV9zdWJtb2R1bGVzlF2UjAtfX2dsb2JhbHNfX5R9lHWGlIZSMC4="
+    }
 }

ppo-Pendulum-v1/policy.optimizer.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f35140a14d394d2df394225b62bcac4e8f0baf3c5e7b6bee35b6a8f07481327
-size 82096
+oid sha256:38a97ba3110e2c7500d7371d0b6d4cae11d9458e713b3558c89bf99b2f55872b
+size 82160

ppo-Pendulum-v1/policy.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c45e5dbe7d88b14202ead17e017d9e6b39f128b089f3b3ae5840a18d9412aeca
-size 40254
+oid sha256:be542f8bb454365173f9e4ea14ab68cabc11b968254a49fb0dbad55bed400c16
+size 40382

ppo-Pendulum-v1/system_info.txt

@@ -1,7 +1,9 @@
-OS: Linux-5.4.0-122-generic-x86_64-with-glibc2.29 #138-Ubuntu SMP Wed Jun 22 15:00:31 UTC 2022
-Python: 3.8.10
-Stable-Baselines3: 1.6.0
-PyTorch: 1.11.0+cu102
-GPU Enabled: False
-Numpy: 1.22.3
-Gym: 0.21.0
+- OS: Linux-5.4.0-156-generic-x86_64-with-glibc2.29 # 173-Ubuntu SMP Tue Jul 11 07:25:22 UTC 2023
+- Python: 3.8.10
+- Stable-Baselines3: 2.2.0a3
+- PyTorch: 2.0.1+cu117
+- GPU Enabled: False
+- Numpy: 1.24.4
+- Cloudpickle: 2.2.1
+- Gymnasium: 0.29.1
+- OpenAI Gym: 0.21.0

replay.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8aff0086945e50d3cb3a42e8a95051e42022f276ab13fdd7114ec2fc3a2d2e7
-size 140831
+oid sha256:cc4ccb77c13e09f524899e42ab016875f30055b24e76e4f43eca6047088e5f14
+size 174142

results.json

@@ -1 +1 @@
-{"mean_reward": -336.8904653, "std_reward": 406.3584272686389, "is_deterministic": true, "n_eval_episodes": 10, "eval_datetime": "2022-07-29T16:39:51.603484"}
+{"mean_reward": -189.2534682, "std_reward": 66.36133815611241, "is_deterministic": true, "n_eval_episodes": 10, "eval_datetime": "2023-09-19T11:41:58.911400"}

train_eval_metrics.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6168a606f3d16b969a034c584d66d5fc4628639ef9e1a564f7c944a5681b27f9
-size 15560
+oid sha256:690af8420de0d7622537e219353fd187a2691b8cf017e74df2f93166a8b7d477
+size 15240