done?

runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: false
+  nstep: 128
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/hydra.yaml

@@ -0,0 +1,155 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=128
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=128
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-04-24_buffer.nstep=128
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/overrides.yaml

@@ -0,0 +1 @@
+- buffer.nstep=128

runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.meta.json

@@ -0,0 +1 @@
+{"step_id": 0, "episode_id": 0, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.meta.json

@@ -0,0 +1 @@
+{"step_id": 371, "episode_id": 2, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-4.meta.json

@@ -0,0 +1 @@
+{"step_id": 740, "episode_id": 4, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.meta.json

@@ -0,0 +1 @@
+{"step_id": 0, "episode_id": 0, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.meta.json

@@ -0,0 +1 @@
+{"step_id": 18, "episode_id": 2, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.meta.json

@@ -0,0 +1 @@
+{"step_id": 36, "episode_id": 4, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.meta.json

@@ -0,0 +1 @@
+{"step_id": 56, "episode_id": 6, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.meta.json

@@ -0,0 +1 @@
+{"step_id": 75, "episode_id": 8, "content_type": "video/mp4"}

runs/2025-10-12/00-04-24_buffer.nstep=128/main.log

@@ -0,0 +1,29 @@
+[2025-10-12 00:04:26,341][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer
+[2025-10-12 00:04:35,294][core][INFO] - Step: 2000, Eval mean: 9.3, Eval std: 0.7810249675906655
+[2025-10-12 00:04:46,380][core][INFO] - Step: 4000, Eval mean: 9.2, Eval std: 0.6
+[2025-10-12 00:04:57,225][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6
+[2025-10-12 00:05:08,588][core][INFO] - Step: 8000, Eval mean: 39.8, Eval std: 15.561490931141527
+[2025-10-12 00:05:20,710][core][INFO] - Step: 10000, Eval mean: 152.7, Eval std: 8.271033792700885
+[2025-10-12 00:05:32,917][core][INFO] - Step: 12000, Eval mean: 107.3, Eval std: 2.934280150224242
+[2025-10-12 00:05:45,065][core][INFO] - Step: 14000, Eval mean: 84.2, Eval std: 3.370459909270544
+[2025-10-12 00:05:57,332][core][INFO] - Step: 16000, Eval mean: 27.0, Eval std: 17.72004514666935
+[2025-10-12 00:06:10,899][core][INFO] - Step: 18000, Eval mean: 184.2, Eval std: 5.035871324805668
+[2025-10-12 00:06:24,745][core][INFO] - Step: 20000, Eval mean: 149.0, Eval std: 14.628738838327793
+[2025-10-12 00:06:37,717][core][INFO] - Step: 22000, Eval mean: 9.7, Eval std: 0.45825756949558394
+[2025-10-12 00:06:50,177][core][INFO] - Step: 24000, Eval mean: 30.1, Eval std: 35.2319457311117
+[2025-10-12 00:07:03,038][core][INFO] - Step: 26000, Eval mean: 53.0, Eval std: 41.6293165929973
+[2025-10-12 00:07:16,211][core][INFO] - Step: 28000, Eval mean: 93.9, Eval std: 4.205948168962618
+[2025-10-12 00:07:28,808][core][INFO] - Step: 30000, Eval mean: 9.4, Eval std: 0.66332495807108
+[2025-10-12 00:07:41,971][core][INFO] - Step: 32000, Eval mean: 9.4, Eval std: 0.66332495807108
+[2025-10-12 00:07:54,891][core][INFO] - Step: 34000, Eval mean: 9.4, Eval std: 0.66332495807108
+[2025-10-12 00:08:08,419][core][INFO] - Step: 36000, Eval mean: 92.3, Eval std: 20.08506908128523
+[2025-10-12 00:08:22,453][core][INFO] - Step: 38000, Eval mean: 108.4, Eval std: 3.2924155266308652
+[2025-10-12 00:08:35,688][core][INFO] - Step: 40000, Eval mean: 9.3, Eval std: 0.45825756949558394
+[2025-10-12 00:08:49,037][core][INFO] - Step: 42000, Eval mean: 10.3, Eval std: 0.6403124237432849
+[2025-10-12 00:09:02,764][core][INFO] - Step: 44000, Eval mean: 55.3, Eval std: 43.91821945388952
+[2025-10-12 00:09:16,935][core][INFO] - Step: 46000, Eval mean: 9.4, Eval std: 0.66332495807108
+[2025-10-12 00:09:31,135][core][INFO] - Step: 48000, Eval mean: 66.0, Eval std: 35.608987629529715
+[2025-10-12 00:09:45,275][core][INFO] - Step: 50000, Eval mean: 9.4, Eval std: 0.66332495807108
+[2025-10-12 00:09:48,740][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\gymnasium\wrappers\monitoring\video_recorder.py:182: UserWarning: WARN: Unable to save last video! Did you call close()?
+  logger.warn("Unable to save last video! Did you call close()?")
+

runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: false
+  nstep: 128
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/hydra.yaml

@@ -0,0 +1,155 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=128
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=128
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-13-09_buffer.nstep=128
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/overrides.yaml

@@ -0,0 +1 @@
+- buffer.nstep=128

runs/2025-10-12/00-13-09_buffer.nstep=128/main.log

@@ -0,0 +1 @@
+[2025-10-12 00:13:11,183][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer

runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: false
+  nstep: 128
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/hydra.yaml

@@ -0,0 +1,155 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=128
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=128
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-15-00_buffer.nstep=128
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/overrides.yaml

@@ -0,0 +1 @@
+- buffer.nstep=128

runs/2025-10-12/00-15-00_buffer.nstep=128/main.log

@@ -0,0 +1 @@
+[2025-10-12 00:15:02,316][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer

runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: false
+  nstep: 128
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/hydra.yaml

@@ -0,0 +1,155 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=128
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=128
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-17-20_buffer.nstep=128
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/overrides.yaml

@@ -0,0 +1 @@
+- buffer.nstep=128

runs/2025-10-12/00-17-20_buffer.nstep=128/main.log

@@ -0,0 +1,16 @@
+[2025-10-12 00:17:22,068][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer
+[2025-10-12 00:17:31,416][core][INFO] - Step: 2000, Eval mean: 9.7, Eval std: 0.7810249675906655
+[2025-10-12 00:17:42,705][core][INFO] - Step: 4000, Eval mean: 10.6, Eval std: 0.66332495807108
+[2025-10-12 00:17:56,752][core][INFO] - Step: 6000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:18:10,715][core][INFO] - Step: 8000, Eval mean: 451.3, Eval std: 146.1
+[2025-10-12 00:18:22,223][core][INFO] - Step: 10000, Eval mean: 9.4, Eval std: 0.4898979485566356
+[2025-10-12 00:18:34,416][core][INFO] - Step: 12000, Eval mean: 9.2, Eval std: 0.6
+[2025-10-12 00:18:49,112][core][INFO] - Step: 14000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:19:04,011][core][INFO] - Step: 16000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:19:16,822][core][INFO] - Step: 18000, Eval mean: 76.1, Eval std: 20.772337374498807
+[2025-10-12 00:19:29,238][core][INFO] - Step: 20000, Eval mean: 9.2, Eval std: 0.6
+[2025-10-12 00:19:45,194][core][INFO] - Step: 22000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:19:58,108][core][INFO] - Step: 24000, Eval mean: 10.4, Eval std: 0.66332495807108
+[2025-10-12 00:20:11,081][core][INFO] - Step: 26000, Eval mean: 9.2, Eval std: 0.6
+[2025-10-12 00:20:26,324][core][INFO] - Step: 28000, Eval mean: 292.1, Eval std: 121.72300522087022
+[2025-10-12 00:20:40,403][core][INFO] - Step: 30000, Eval mean: 37.1, Eval std: 39.459979726299906

runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: true
+  nstep: 16
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml

@@ -0,0 +1,156 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=16
+    - buffer.use_per=true
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=16,buffer.use_per=true
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-21-15_buffer.nstep=16,buffer.use_per=true
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- buffer.nstep=16
+- buffer.use_per=true

runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: true
+  nstep: 16
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml

@@ -0,0 +1,156 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=16
+    - buffer.use_per=true
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=16,buffer.use_per=true
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-21-37_buffer.nstep=16,buffer.use_per=true
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- buffer.nstep=16
+- buffer.use_per=true

runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/main.log

@@ -0,0 +1 @@
+[2025-10-12 00:21:39,758][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized16StepReplayBuffer

runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: true
+  nstep: 16
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml

@@ -0,0 +1,156 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=16
+    - buffer.use_per=true
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=16,buffer.use_per=true
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-22-14_buffer.nstep=16,buffer.use_per=true
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- buffer.nstep=16
+- buffer.use_per=true

runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/main.log

@@ -0,0 +1,13 @@
+[2025-10-12 00:22:15,881][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized16StepReplayBuffer
+[2025-10-12 00:22:32,257][core][INFO] - Step: 2000, Eval mean: 23.1, Eval std: 10.921996154549772
+[2025-10-12 00:22:50,791][core][INFO] - Step: 4000, Eval mean: 10.3, Eval std: 1.2688577540449522
+[2025-10-12 00:23:08,418][core][INFO] - Step: 6000, Eval mean: 13.5, Eval std: 2.247220505424423
+[2025-10-12 00:23:26,620][core][INFO] - Step: 8000, Eval mean: 13.0, Eval std: 1.2649110640673518
+[2025-10-12 00:23:44,856][core][INFO] - Step: 10000, Eval mean: 12.3, Eval std: 0.9
+[2025-10-12 00:24:03,104][core][INFO] - Step: 12000, Eval mean: 13.0, Eval std: 1.7888543819998317
+[2025-10-12 00:24:21,653][core][INFO] - Step: 14000, Eval mean: 11.1, Eval std: 3.1448370387032774
+[2025-10-12 00:24:40,292][core][INFO] - Step: 16000, Eval mean: 14.6, Eval std: 3.6386810797320503
+[2025-10-12 00:24:58,872][core][INFO] - Step: 18000, Eval mean: 13.9, Eval std: 1.6401219466856727
+[2025-10-12 00:25:18,000][core][INFO] - Step: 20000, Eval mean: 26.4, Eval std: 13.462540622037135
+[2025-10-12 00:25:36,671][core][INFO] - Step: 22000, Eval mean: 42.0, Eval std: 23.112767034693185
+[2025-10-12 00:25:54,993][core][INFO] - Step: 24000, Eval mean: 12.8, Eval std: 1.6613247725836149

runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: true
+  nstep: 10
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/hydra.yaml

@@ -0,0 +1,156 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=10
+    - buffer.use_per=true
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=10,buffer.use_per=true
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-26-19_buffer.nstep=10,buffer.use_per=true
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- buffer.nstep=10
+- buffer.use_per=true

runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/main.log

@@ -0,0 +1,25 @@
+[2025-10-12 00:26:21,193][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized10StepReplayBuffer
+[2025-10-12 00:26:37,205][core][INFO] - Step: 2000, Eval mean: 12.2, Eval std: 0.8717797887081347
+[2025-10-12 00:26:54,645][core][INFO] - Step: 4000, Eval mean: 9.6, Eval std: 1.2806248474865698
+[2025-10-12 00:27:12,056][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6
+[2025-10-12 00:27:29,842][core][INFO] - Step: 8000, Eval mean: 21.6, Eval std: 5.730619512757761
+[2025-10-12 00:27:48,840][core][INFO] - Step: 10000, Eval mean: 227.8, Eval std: 182.14543639630392
+[2025-10-12 00:28:09,044][core][INFO] - Step: 12000, Eval mean: 432.6, Eval std: 135.09641001891944
+[2025-10-12 00:28:29,323][core][INFO] - Step: 14000, Eval mean: 360.9, Eval std: 121.26866866590068
+[2025-10-12 00:28:49,316][core][INFO] - Step: 16000, Eval mean: 368.7, Eval std: 105.315763302556
+[2025-10-12 00:29:09,905][core][INFO] - Step: 18000, Eval mean: 370.6, Eval std: 114.3268997218065
+[2025-10-12 00:29:30,078][core][INFO] - Step: 20000, Eval mean: 380.4, Eval std: 108.37545847653887
+[2025-10-12 00:29:50,352][core][INFO] - Step: 22000, Eval mean: 436.0, Eval std: 89.43489251964247
+[2025-10-12 00:30:10,478][core][INFO] - Step: 24000, Eval mean: 437.1, Eval std: 93.34286260877154
+[2025-10-12 00:30:30,447][core][INFO] - Step: 26000, Eval mean: 451.8, Eval std: 83.40119903214821
+[2025-10-12 00:30:50,912][core][INFO] - Step: 28000, Eval mean: 439.1, Eval std: 85.54115968351142
+[2025-10-12 00:31:11,646][core][INFO] - Step: 30000, Eval mean: 454.0, Eval std: 69.80257874892588
+[2025-10-12 00:31:32,031][core][INFO] - Step: 32000, Eval mean: 423.2, Eval std: 75.24333857558422
+[2025-10-12 00:31:52,381][core][INFO] - Step: 34000, Eval mean: 386.5, Eval std: 103.64386137152552
+[2025-10-12 00:32:12,858][core][INFO] - Step: 36000, Eval mean: 456.8, Eval std: 84.32057874564192
+[2025-10-12 00:32:33,233][core][INFO] - Step: 38000, Eval mean: 448.7, Eval std: 96.5691979877642
+[2025-10-12 00:32:53,194][core][INFO] - Step: 40000, Eval mean: 455.4, Eval std: 83.44123680770797
+[2025-10-12 00:33:13,862][core][INFO] - Step: 42000, Eval mean: 431.9, Eval std: 89.02634441557173
+[2025-10-12 00:33:34,167][core][INFO] - Step: 44000, Eval mean: 426.1, Eval std: 91.27370924861113
+[2025-10-12 00:33:54,320][core][INFO] - Step: 46000, Eval mean: 422.8, Eval std: 84.57635603405954
+[2025-10-12 00:34:14,480][core][INFO] - Step: 48000, Eval mean: 439.7, Eval std: 94.10850121003946

runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/config.yaml

@@ -0,0 +1,33 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+buffer:
+  capacity: 50000
+  use_per: true
+  nstep: 5
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/hydra.yaml

@@ -0,0 +1,156 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - buffer.nstep=5
+    - buffer.use_per=true
+  job:
+    name: main
+    chdir: true
+    override_dirname: buffer.nstep=5,buffer.use_per=true
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-34-29_buffer.nstep=5,buffer.use_per=true
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- buffer.nstep=5
+- buffer.use_per=true

runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/main.log

@@ -0,0 +1,28 @@
+[2025-10-12 00:34:31,512][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized5StepReplayBuffer
+[2025-10-12 00:34:50,453][core][INFO] - Step: 2000, Eval mean: 442.7, Eval std: 115.3568810257975
+[2025-10-12 00:35:10,869][core][INFO] - Step: 4000, Eval mean: 449.6, Eval std: 101.87462883367968
+[2025-10-12 00:35:30,844][core][INFO] - Step: 6000, Eval mean: 367.7, Eval std: 101.56972974267482
+[2025-10-12 00:35:50,826][core][INFO] - Step: 8000, Eval mean: 284.6, Eval std: 37.296648643008126
+[2025-10-12 00:36:10,180][core][INFO] - Step: 10000, Eval mean: 224.7, Eval std: 18.10552401892859
+[2025-10-12 00:36:30,662][core][INFO] - Step: 12000, Eval mean: 317.9, Eval std: 68.10058736897943
+[2025-10-12 00:36:49,780][core][INFO] - Step: 14000, Eval mean: 156.4, Eval std: 7.255342858886822
+[2025-10-12 00:37:09,606][core][INFO] - Step: 16000, Eval mean: 213.9, Eval std: 15.286922515666781
+[2025-10-12 00:37:31,528][core][INFO] - Step: 18000, Eval mean: 489.4, Eval std: 31.8
+[2025-10-12 00:37:51,012][core][INFO] - Step: 20000, Eval mean: 148.8, Eval std: 6.446704584514478
+[2025-10-12 00:38:12,456][core][INFO] - Step: 22000, Eval mean: 475.1, Eval std: 74.7
+[2025-10-12 00:38:31,795][core][INFO] - Step: 24000, Eval mean: 156.9, Eval std: 9.235258523723092
+[2025-10-12 00:38:51,389][core][INFO] - Step: 26000, Eval mean: 201.0, Eval std: 9.02219485491197
+[2025-10-12 00:39:12,304][core][INFO] - Step: 28000, Eval mean: 487.5, Eval std: 37.5
+[2025-10-12 00:39:33,586][core][INFO] - Step: 30000, Eval mean: 475.4, Eval std: 73.8
+[2025-10-12 00:39:54,512][core][INFO] - Step: 32000, Eval mean: 486.2, Eval std: 41.4
+[2025-10-12 00:40:13,757][core][INFO] - Step: 34000, Eval mean: 129.1, Eval std: 6.122907805936653
+[2025-10-12 00:40:35,414][core][INFO] - Step: 36000, Eval mean: 488.8, Eval std: 33.6
+[2025-10-12 00:40:57,324][core][INFO] - Step: 38000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:41:16,949][core][INFO] - Step: 40000, Eval mean: 225.3, Eval std: 20.459960899278375
+[2025-10-12 00:41:37,212][core][INFO] - Step: 42000, Eval mean: 209.4, Eval std: 8.321057625085889
+[2025-10-12 00:41:57,108][core][INFO] - Step: 44000, Eval mean: 204.2, Eval std: 10.934349546269315
+[2025-10-12 00:42:18,230][core][INFO] - Step: 46000, Eval mean: 470.0, Eval std: 60.13484846576068
+[2025-10-12 00:42:38,199][core][INFO] - Step: 48000, Eval mean: 181.5, Eval std: 11.351211389098522
+[2025-10-12 00:43:00,221][core][INFO] - Step: 50000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:43:23,166][core][INFO] - Final Eval mean: 500.0, Eval std: 0.0
+[2025-10-12 00:43:29,422][__main__][INFO] - Finish training with eval mean: 500.0