updates

runs/Noisy DQN/.hydra/config.yaml

@@ -0,0 +1,35 @@
+seed: 42
+env_name: CartPole-v1
+train:
+  nstep: ${buffer.nstep}
+  timesteps: 50000
+  batch_size: 128
+  test_every: 2500
+  eps_max: 1
+  eps_min: 0.05
+  eps_steps: 12500
+  start_steps: 0
+  plot_interval: 2000
+  eval_interval: 2000
+  eval_episodes: 10
+agent:
+  gamma: 0.99
+  lr: 0.002
+  tau: 0.1
+  nstep: ${buffer.nstep}
+  target_update_interval: 3
+  hidden_size: 64
+  activation:
+    _target_: torch.nn.ELU
+  use_dueling: false
+  use_double: false
+  use_noisy: true
+  noisy_sigma: 0.017
+buffer:
+  capacity: 50000
+  use_per: false
+  nstep: 1
+  gamma: ${agent.gamma}
+  per_alpha: 0.7
+  per_beta: 0.4
+  per_eps: 0.01

runs/Noisy DQN/.hydra/hydra.yaml

@@ -0,0 +1,156 @@
+hydra:
+  run:
+    dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+  sweep:
+    dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent.use_noisy=true
+    - agent.noisy_sigma=0.017
+  job:
+    name: main
+    chdir: true
+    override_dirname: agent.noisy_sigma=0.017,agent.use_noisy=true
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-14\20-10-21_agent.noisy_sigma=0.017,agent.use_noisy=true
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

runs/Noisy DQN/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- agent.use_noisy=true
+- agent.noisy_sigma=0.017

runs/Noisy DQN/main.log

@@ -0,0 +1,28 @@
+[2025-10-14 20:10:23,212][__main__][INFO] - Training for 50000 timesteps with NoisyQNetworkwith noisy sigma=0.017 and NormalReplayBuffer
+[2025-10-14 20:10:37,802][core][INFO] - Step: 2000, Eval mean: 198.8, Eval std: 44.11757019601148
+[2025-10-14 20:10:54,273][core][INFO] - Step: 4000, Eval mean: 232.8, Eval std: 62.62874739287063
+[2025-10-14 20:11:11,664][core][INFO] - Step: 6000, Eval mean: 315.8, Eval std: 59.45889336339855
+[2025-10-14 20:11:29,420][core][INFO] - Step: 8000, Eval mean: 333.8, Eval std: 37.796296114831144
+[2025-10-14 20:11:49,484][core][INFO] - Step: 10000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:12:06,677][core][INFO] - Step: 12000, Eval mean: 173.5, Eval std: 15.422386326376342
+[2025-10-14 20:12:23,142][core][INFO] - Step: 14000, Eval mean: 117.6, Eval std: 2.90516780926679
+[2025-10-14 20:12:43,060][core][INFO] - Step: 16000, Eval mean: 476.0, Eval std: 62.509199322979654
+[2025-10-14 20:13:03,825][core][INFO] - Step: 18000, Eval mean: 470.9, Eval std: 32.28141880401169
+[2025-10-14 20:13:24,671][core][INFO] - Step: 20000, Eval mean: 456.8, Eval std: 37.10202150826825
+[2025-10-14 20:13:44,654][core][INFO] - Step: 22000, Eval mean: 351.5, Eval std: 91.09253536926064
+[2025-10-14 20:14:03,339][core][INFO] - Step: 24000, Eval mean: 138.2, Eval std: 2.891366458960192
+[2025-10-14 20:14:25,926][core][INFO] - Step: 26000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:14:48,281][core][INFO] - Step: 28000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:15:11,898][core][INFO] - Step: 30000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:15:34,603][core][INFO] - Step: 32000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:15:55,926][core][INFO] - Step: 34000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:16:17,190][core][INFO] - Step: 36000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:16:35,708][core][INFO] - Step: 38000, Eval mean: 212.6, Eval std: 22.698898651696737
+[2025-10-14 20:16:57,155][core][INFO] - Step: 40000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:17:19,003][core][INFO] - Step: 42000, Eval mean: 483.7, Eval std: 31.189902212094225
+[2025-10-14 20:17:40,610][core][INFO] - Step: 44000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:18:03,087][core][INFO] - Step: 46000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:18:25,316][core][INFO] - Step: 48000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:18:47,463][core][INFO] - Step: 50000, Eval mean: 500.0, Eval std: 0.0
+[2025-10-14 20:19:12,165][core][INFO] - Final Eval mean: 491.8, Eval std: 24.599999999999998
+[2025-10-14 20:19:17,782][__main__][INFO] - Finish training with eval mean: 491.8