commit 250f763f1f507a166f99aeb1cae00d52ba38e152 Author: Zheyuan Wu <60459821+Trance-0@users.noreply.github.com> Date: Sun Oct 12 00:55:07 2025 -0500 done? diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a160d09 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# python files +__pycache__ \ No newline at end of file diff --git a/hw2/DRL_Homework_2.pdf b/hw2/DRL_Homework_2.pdf new file mode 100644 index 0000000..c60f7b5 Binary files /dev/null and b/hw2/DRL_Homework_2.pdf differ diff --git a/hw2/README.md b/hw2/README.md new file mode 100644 index 0000000..1169c5a --- /dev/null +++ b/hw2/README.md @@ -0,0 +1,24 @@ +# Installation + +Since we are using PyTorch for hw2, we recommend using conda to manage the environment. Please refer to the [miniconda](https://docs.conda.io/en/latest/miniconda.html) homepage for a compact conda installation. + +You have two options for creating the environment of hw2 + +* For mac users or a cpu-only installation, please remove the `pytorch-cuda` term in either ways. +* To create a new conda environment, simply run `conda env create -f environment.yml` +* If you want to install the package within the environment you created with hw1, please following the below steps: + + ```bash + conda activate + # we are using PyTorch 2.0! + # remove the pytorch-cuda=11.7 term if you are a mac user to want a cpu-only installation + conda install pytorch==2.0.0 pytorch-cuda=11.7 -c pytorch -c nvidia + pip install gymnasium[classic_control]==0.27.1 + pip install matplotlib==3.7.1 + # for hyperparameter management + pip install hydra-core==1.3.2 + # for video recording + pip install moviepy==1.0.3 + ``` + +That's it! If you encounter any trouble creating the environment, please let us know :-) diff --git a/hw2/agent.py b/hw2/agent.py new file mode 100644 index 0000000..25c89fb --- /dev/null +++ b/hw2/agent.py @@ -0,0 +1,127 @@ +import os +import torch +import torch.optim as optim +from copy import deepcopy +from model import QNetwork, DuelingQNetwork +from gymnasium.wrappers import TimeLimit + +class DQNAgent: + def __init__(self, state_size, action_size, cfg, device='cuda'): + self.device = device + self.use_double = cfg.use_double + self.use_dueling = cfg.use_dueling + self.target_update_interval = cfg.target_update_interval + q_model = DuelingQNetwork if self.use_dueling else QNetwork + + self.q_net = q_model(state_size, action_size, cfg.hidden_size, cfg.activation).to(self.device) + self.target_net = deepcopy(self.q_net).to(self.device) + self.optimizer = optim.AdamW(self.q_net.parameters(), lr=cfg.lr) + + self.tau = cfg.tau + # update the gamma we use in the Bellman equation for n-step DQN + self.gamma = cfg.gamma ** cfg.nstep + + def soft_update(self, target, source): + """ + Soft update the target network using the source network + """ + for target_param, source_param in zip(target.parameters(), source.parameters()): + target_param.data.copy_((1 - self.tau) * target_param.data + self.tau * source_param.data) + + @torch.no_grad() + def get_action(self, state): + """ + Get the action according to the current state and Q value + """ + + ############################ + # YOUR IMPLEMENTATION HERE # + + # update from single state + torch_max_idx = torch.argmax(self.q_net(torch.tensor(state).to(self.device)), dim=0) + action = torch_max_idx.cpu().numpy() + ############################ + return action + + @torch.no_grad() + def get_Q_target(self, state, action, reward, done, next_state) -> torch.Tensor: + """ + Get the target Q value according to the Bellman equation + """ + if self.use_double: + # YOUR IMPLEMENTATION HERE + reward_tensor = reward.to(self.device) + # update from batch states via q_net + next_q_tensor = self.q_net(next_state.to(self.device)) + # return the max Q value + next_q = torch.max(next_q_tensor, dim=1).values + q_target = reward_tensor + (1-done.to(self.device)) * self.gamma * next_q + return q_target + else: + # YOUR IMPLEMENTATION HERE + reward_tensor = reward.to(self.device) + # update from batch states + next_q_tensor = self.target_net(next_state.to(self.device)) + # return the max Q value + next_q = torch.max(next_q_tensor, dim=1).values + q_target = reward_tensor + (1-done.to(self.device)) * self.gamma * next_q + return q_target + + def get_Q(self, state, action, use_double_net=False) -> torch.Tensor: + """ + Get the Q value of the current state and action + """ + ############################ + # YOUR IMPLEMENTATION HERE # + if use_double_net: + # get from target net + q_tensor = self.target_net(state.to(self.device)) + action_idx = action.squeeze(1).to(dtype=torch.int32).to(self.device) + # select corresponding action, do not use index_select... That don't works + q = q_tensor.gather(1, action_idx.unsqueeze(1)).squeeze(1) + return q + else: + # elegant python move by Jack Wu. Fantastic... + # q= self.q_net(state.to(self.device))[:, action.int()] + # update from batch states + q_tensor = self.q_net(state.to(self.device)) + action_idx = action.squeeze(1).to(dtype=torch.int32).to(self.device) + # select corresponding action, do not use index_select... That don't works + q = q_tensor.gather(1, action_idx.unsqueeze(1)).squeeze(1) + return q + ############################ + + def update(self, batch, step, weights=None): + state, action, reward, next_state, done = batch + + Q_target = self.get_Q_target(state, action, reward, done, next_state) + + Q = self.get_Q(state, action) + + if weights is None: + weights = torch.ones_like(Q).to(self.device) + + td_error = torch.abs(Q - Q_target).detach() + loss = torch.mean((Q - Q_target)**2 * weights) + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + if not step % self.target_update_interval: + with torch.no_grad(): + self.soft_update(self.target_net, self.q_net) + + return loss.item(), td_error, Q.mean().item() + + def save(self, name): + os.makedirs('models', exist_ok=True) + torch.save(self.q_net.state_dict(), os.path.join('models', name)) + + def load(self, name='best.pt'): + self.q_net.load_state_dict(torch.load(os.path.join('models', name))) + + def __repr__(self) -> str: + use_double = 'Double' if self.use_double else '' + use_dueling = 'Dueling' if self.use_dueling else '' + prefix = 'Normal' if not self.use_double and not self.use_dueling else '' + return use_double + use_dueling + prefix + 'QNetwork' diff --git a/hw2/buffer.py b/hw2/buffer.py new file mode 100644 index 0000000..19f4b71 --- /dev/null +++ b/hw2/buffer.py @@ -0,0 +1,202 @@ +import torch +import numpy as np +from collections import deque + + +def get_buffer(cfg, **args): + assert type(cfg.nstep) == int and cfg.nstep > 0, 'nstep must be a positive integer' + if not cfg.use_per: + if cfg.nstep == 1: + return ReplayBuffer(cfg.capacity, **args) + else: + return NStepReplayBuffer(cfg.capacity, cfg.nstep, cfg.gamma, **args) + else: + if cfg.nstep == 1: + return PrioritizedReplayBuffer(cfg.capacity, cfg.per_eps, cfg.per_alpha, cfg.per_beta, **args) + else: + return PrioritizedNStepReplayBuffer(cfg.capacity, cfg.per_eps, cfg.per_alpha, cfg.per_beta, cfg.nstep, cfg.gamma, **args) + + +class ReplayBuffer: + def __init__(self, capacity, state_size, device): + self.device = device + self.state = torch.empty(capacity, state_size, dtype=torch.float) + self.action = torch.empty(capacity, 1, dtype=torch.float) + self.reward = torch.empty(capacity, dtype=torch.float) + self.next_state = torch.empty(capacity, state_size, dtype=torch.float) + self.done = torch.empty(capacity, dtype=torch.int) + + self.idx = 0 + self.size = 0 + self.capacity = capacity + + def __repr__(self) -> str: + return 'NormalReplayBuffer' + + def add(self, transition): + state, action, reward, next_state, done = transition + + # store transition in the buffer and update the index and size of the buffer + # you may need to convert the data type to torch.tensor + + ############################ + # YOUR IMPLEMENTATION HERE # + self.state[self.idx] = torch.tensor(state, device=self.device) + self.action[self.idx] = torch.tensor(action, device=self.device) + self.reward[self.idx] = torch.tensor(reward, device=self.device) + self.next_state[self.idx] = torch.tensor(next_state, device=self.device) + self.done[self.idx] = torch.tensor(done, device=self.device) + self.idx = (self.idx + 1) % self.capacity + self.size = min(self.size + 1, self.capacity) + + ############################ + + def sample(self, batch_size): + # sample batch_size data from the buffer without replacement + sample_idxs = np.random.choice(self.size, batch_size, replace=False) + batch = () + # get a batch of data from the buffer according to the sample_idxs + # please transfer the data to the corresponding device before return + ############################ + # YOUR IMPLEMENTATION HERE # + # do not load to gpu device since the buffer is not loaded on init + batch = (torch.index_select(self.state, 0, torch.tensor(sample_idxs)), + torch.index_select(self.action, 0, torch.tensor(sample_idxs)), + torch.index_select(self.reward, 0, torch.tensor(sample_idxs)), + torch.index_select(self.next_state, 0, torch.tensor(sample_idxs)), + torch.index_select(self.done, 0, torch.tensor(sample_idxs)) + ) + + ############################ + return batch + + +class NStepReplayBuffer(ReplayBuffer): + def __init__(self, capacity, n_step, gamma, state_size, device): + super().__init__(capacity, state_size, device=device) + self.n_step = n_step + self.n_step_buffer = deque([], maxlen=n_step) + self.gamma = gamma + + def __repr__(self) -> str: + return f'{self.n_step}StepReplayBuffer' + + def n_step_handler(self): + """Get n-step state, action, reward and done for the transition, discard those rewards after done=True""" + ############################ + # YOUR IMPLEMENTATION HERE # + state, action, reward, done = self.n_step_buffer[0] + # compute n-step discounted reward + gamma = self.gamma + for i in range(1, len(self.n_step_buffer)): + if done: + break + reward += gamma * self.n_step_buffer[i][2] + gamma *= self.gamma + ############################ + return state, action, reward, done + + def add(self, transition): + state, action, reward, next_state, done = transition + self.n_step_buffer.append((state, action, reward, done)) + if len(self.n_step_buffer) < self.n_step: + return + state, action, reward, done = self.n_step_handler() + super().add((state, action, reward, next_state, done)) + + +class PrioritizedReplayBuffer(ReplayBuffer): + def __init__(self, capacity, eps, alpha, beta, state_size, device): + self.weights = np.zeros(capacity, dtype=np.float32) # stores weights for importance sampling + self.eps = eps # minimal priority for stability + self.alpha = alpha # determines how much prioritization is used, α = 0 corresponding to the uniform case + self.beta = beta # determines the amount of importance-sampling correction, b = 1 fully compensate for the non-uniform probabilities + self.max_priority = eps # priority for new samples, init as eps + super().__init__(capacity, state_size, device=device) + + def add(self, transition): + """ + Add a new experience to memory, and update it's priority to the max_priority. + """ + ############################ + # YOUR IMPLEMENTATION HERE # + super().add(transition) + self.weights[self.idx] = self.max_priority + ############################ + + + def sample(self, batch_size): + """ + Sample a batch of experiences from the buffer with priority, and calculates the weights used for the correction of bias used in the Q-learning update + Returns: + batch: a batch of experiences as in the normal replay buffer + weights: torch.Tensor (batch_size, ), importance sampling weights for each sample + sample_idxs: numpy.ndarray (batch_size, ), the indexes of the sample in the buffer + """ + ############################ + # YOUR IMPLEMENTATION HERE # + # assume sample with replacement, in case if sample size is too small + sample_idxs_tensor = torch.multinomial(torch.tensor(self.weights), batch_size, replacement=True) + sample_idxs = sample_idxs_tensor.cpu().numpy() + # do not load to gpu device since the buffer is not loaded on init + batch = ( + torch.index_select(self.state, 0, torch.tensor(sample_idxs)), + torch.index_select(self.action, 0, torch.tensor(sample_idxs)), + torch.index_select(self.reward, 0, torch.tensor(sample_idxs)), + torch.index_select(self.next_state, 0, torch.tensor(sample_idxs)), + torch.index_select(self.done, 0, torch.tensor(sample_idxs)) + ) + weights = torch.tensor(self.weights[sample_idxs], device=self.device).unsqueeze(1) + ############################ + return batch, weights, sample_idxs + + def update_priorities(self, data_idxs, priorities: np.ndarray): + priorities = (priorities + self.eps) ** self.alpha + + self.weights[data_idxs] = priorities + self.max_priority = max(self.weights) + + def __repr__(self) -> str: + return 'PrioritizedReplayBuffer' + + +# Avoid Diamond Inheritance +class PrioritizedNStepReplayBuffer(PrioritizedReplayBuffer): + def __init__(self, capacity, eps, alpha, beta, n_step, gamma, state_size, device): + ############################ + # YOUR IMPLEMENTATION HERE # + super().__init__(capacity, eps, alpha, beta, state_size, device) + self.n_step = n_step + self.n_step_buffer = deque([], maxlen=n_step) + self.gamma = gamma + ############################ + def __repr__(self) -> str: + return f'Prioritized{self.n_step}StepReplayBuffer' + + def add(self, transition): + ############################ + # YOUR IMPLEMENTATION HERE # + state, action, reward, next_state, done = transition + self.n_step_buffer.append((state, action, reward, done)) + if len(self.n_step_buffer) < self.n_step: + return + state, action, reward, done = self.n_step_handler() + super().add((state, action, reward, next_state, done)) + ############################ + + # def the other necessary class methods as your need + + def n_step_handler(self): + """Get n-step state, action, reward and done for the transition, discard those rewards after done=True""" + ############################ + # YOUR IMPLEMENTATION HERE # + state, action, reward, done = self.n_step_buffer[0] + # compute n-step discounted reward + gamma = self.gamma + for i in range(1, len(self.n_step_buffer)): + if done: + break + reward += gamma * self.n_step_buffer[i][2] + gamma *= self.gamma + ############################ + return state, action, reward, done \ No newline at end of file diff --git a/hw2/cfgs/config.yaml b/hw2/cfgs/config.yaml new file mode 100644 index 0000000..31aee82 --- /dev/null +++ b/hw2/cfgs/config.yaml @@ -0,0 +1,45 @@ +seed: 42 +env_name: CartPole-v1 + +train: + nstep: ${buffer.nstep} + timesteps: 50_000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12_500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 + +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + # you can define other parameters of the __init__ function (if any) for the object here + use_dueling: False + use_double: False + +buffer: + capacity: 50_000 + use_per: False + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 + +hydra: + job: + chdir: true + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} \ No newline at end of file diff --git a/hw2/core.py b/hw2/core.py new file mode 100644 index 0000000..8fb8c4b --- /dev/null +++ b/hw2/core.py @@ -0,0 +1,128 @@ +from copy import deepcopy +import random +import logging +import numpy as np +from buffer import ReplayBuffer, PrioritizedReplayBuffer +import matplotlib.pyplot as plt +from utils import moving_average, merge_videos, get_epsilon +from gymnasium.wrappers import RecordVideo, RecordEpisodeStatistics +logger = logging.getLogger(__name__) + + +def visualize(step, title, train_steps, train_returns, eval_steps, eval_returns, losses, q_values): + train_window, loss_window, q_window = 10, 100, 100 + plt.figure(figsize=(20, 6)) + + # plot train and eval returns + plt.subplot(1, 3, 1) + plt.title('frame %s. score: %s' % (step, np.mean(train_returns[-train_window:]))) + plt.plot(train_steps[train_window - 1:], moving_average(train_returns, train_window), label='train') + if len(eval_steps) > 0: + plt.plot(eval_steps, eval_returns, label='eval') + plt.legend() + plt.xlabel('step') + + # plot td losses + plt.subplot(1, 3, 2) + plt.title('loss') + plt.plot(moving_average(losses, loss_window)) + plt.xlabel('step') + plt.subplot(1, 3, 3) + + # plot q values + plt.title('q_values') + plt.plot(moving_average(q_values, q_window)) + plt.xlabel('step') + plt.suptitle(title, fontsize=16) + plt.savefig('results.png') + plt.close() + + +def eval(env, agent, episodes, seed): + returns = [] + for episode in range(episodes): + state, _ = env.reset(seed=episode + seed) + done, truncated = False, False + while not (done or truncated): + state, _, done, truncated, info = env.step(agent.get_action(state)) + + returns.append(info['episode']['r'].item()) + return np.mean(returns), np.std(returns) + + +def train(cfg, env, agent, buffer, seed): + # wrap env to record episode returns + env = RecordEpisodeStatistics(env) + eval_env = deepcopy(env) + losses, Qs = [], [] + episode_rewards, train_steps = [], [] + eval_rewards, eval_steps = [], [] + + best_reward = -np.inf + done, truncated = False, False + state, _ = env.reset(seed=seed) + + for step in range(1, cfg.timesteps + 1): + if done or truncated: + state, _ = env.reset() + done, truncated = False, False + # store episode reward + episode_rewards.append(info['episode']['r'].item()) + train_steps.append(step - 1) + + eps = get_epsilon(step - 1, cfg.eps_min, cfg.eps_max, cfg.eps_steps) + + if random.random() < eps: + action = env.action_space.sample() + else: + action = agent.get_action(state) + + next_state, reward, done, truncated, info = env.step(action) + buffer.add((state, action, reward, next_state, int(done))) + state = next_state + + if step > cfg.batch_size + cfg.nstep: + # sample and do one step update + if isinstance(buffer, PrioritizedReplayBuffer): + # sample with priorities and update the priorities with td_error + batch, weights, tree_idxs = buffer.sample(cfg.batch_size) + loss, td_error, Q = agent.update(batch, step, weights=weights) + + buffer.update_priorities(tree_idxs, td_error.cpu().numpy()) + elif isinstance(buffer, ReplayBuffer): + batch = buffer.sample(cfg.batch_size) + loss, _, Q = agent.update(batch, step) + else: + raise RuntimeError("Unknown Buffer") + + Qs.append(Q) + losses.append(loss) + + if step % cfg.eval_interval == 0: + eval_mean, eval_std = eval(eval_env, agent=agent, episodes=cfg.eval_episodes, seed=seed) + state, _ = env.reset() + eval_steps.append(step - 1) + eval_rewards.append(eval_mean) + logger.info(f"Step: {step}, Eval mean: {eval_mean}, Eval std: {eval_std}") + if eval_mean > best_reward: + best_reward = eval_mean + agent.save('best_model.pt') + + if step % cfg.plot_interval == 0: + visualize(step, f'{agent} with {buffer}', train_steps, episode_rewards, eval_steps, eval_rewards, losses, Qs) + + agent.save('final_model.pt') + visualize(step, f'{agent} with {buffer}', train_steps, episode_rewards, eval_steps, eval_rewards, losses, Qs) + + env = RecordVideo(eval_env, 'final_videos', name_prefix='eval', episode_trigger=lambda x: x % 2 == 0 and x < cfg.eval_episodes) + eval_mean, eval_std = eval(env, agent=agent, episodes=cfg.eval_episodes, seed=seed) + + agent.load('best_model.pt') # use best model for visualization + env = RecordVideo(eval_env, 'best_videos', name_prefix='eval', episode_trigger=lambda x: x % 2 == 0 and x < cfg.eval_episodes) + eval_mean, eval_std = eval(env, agent=agent, episodes=cfg.eval_episodes, seed=seed) + + env.close() + logger.info(f"Final Eval mean: {eval_mean}, Eval std: {eval_std}") + merge_videos('final_videos') + merge_videos('best_videos') + return eval_mean diff --git a/hw2/environment.yml b/hw2/environment.yml new file mode 100644 index 0000000..d7204e1 --- /dev/null +++ b/hw2/environment.yml @@ -0,0 +1,15 @@ +name: drl_hw2 +channels: + - pytorch + - nvidia + - defaults +dependencies: + - python=3.10 + - pytorch=2.0.0 + - pytorch-cuda=11.7 # Comment this line if you are a mac user or want a cpu-only installation + - pip=23.0.1 + - pip: + - gymnasium[classic-control]==0.27.1 + - hydra-core==1.3.2 + - matplotlib==3.7.1 + - moviepy==1.0.3 \ No newline at end of file diff --git a/hw2/gallery/A very stable agent.mp4 b/hw2/gallery/A very stable agent.mp4 new file mode 100644 index 0000000..4396612 Binary files /dev/null and b/hw2/gallery/A very stable agent.mp4 differ diff --git a/hw2/gallery/All-In-One.png b/hw2/gallery/All-In-One.png new file mode 100644 index 0000000..226e30e Binary files /dev/null and b/hw2/gallery/All-In-One.png differ diff --git a/hw2/gallery/DQN.png b/hw2/gallery/DQN.png new file mode 100644 index 0000000..6704897 Binary files /dev/null and b/hw2/gallery/DQN.png differ diff --git a/hw2/gallery/Double DQN.png b/hw2/gallery/Double DQN.png new file mode 100644 index 0000000..5c96018 Binary files /dev/null and b/hw2/gallery/Double DQN.png differ diff --git a/hw2/gallery/Dueling DQN.png b/hw2/gallery/Dueling DQN.png new file mode 100644 index 0000000..7a64f12 Binary files /dev/null and b/hw2/gallery/Dueling DQN.png differ diff --git a/hw2/gallery/NStep + PER.png b/hw2/gallery/NStep + PER.png new file mode 100644 index 0000000..6d1020f Binary files /dev/null and b/hw2/gallery/NStep + PER.png differ diff --git a/hw2/gallery/NStep.png b/hw2/gallery/NStep.png new file mode 100644 index 0000000..0c1bfeb Binary files /dev/null and b/hw2/gallery/NStep.png differ diff --git a/hw2/gallery/PER.png b/hw2/gallery/PER.png new file mode 100644 index 0000000..e13e03b Binary files /dev/null and b/hw2/gallery/PER.png differ diff --git a/hw2/main.py b/hw2/main.py new file mode 100644 index 0000000..613c15c --- /dev/null +++ b/hw2/main.py @@ -0,0 +1,30 @@ +import hydra +import utils +import torch +import logging +from agent import DQNAgent +from core import train +from buffer import get_buffer +import gymnasium as gym +logger = logging.getLogger(__name__) +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +@hydra.main(config_path="cfgs", config_name="config", version_base="1.3") +def main(cfg): + env = gym.make(cfg.env_name, render_mode="rgb_array") + utils.set_seed_everywhere(env, cfg.seed) + + state_size = utils.get_space_shape(env.observation_space) + action_size = utils.get_space_shape(env.action_space) + + buffer = get_buffer(cfg.buffer, state_size=state_size, device=device) + agent = DQNAgent(state_size=state_size, action_size=action_size, cfg=cfg.agent, device=device) + + logger.info(f"Training for {cfg.train.timesteps} timesteps with {agent} and {buffer}") + eval_mean = train(cfg.train, env, agent, buffer, seed=cfg.seed) + logger.info(f"Finish training with eval mean: {eval_mean}") + + +if __name__ == "__main__": + main() diff --git a/hw2/model.py b/hw2/model.py new file mode 100644 index 0000000..72d7a9d --- /dev/null +++ b/hw2/model.py @@ -0,0 +1,53 @@ +from hydra.utils import instantiate +import torch +import torch.nn as nn + + +class QNetwork(nn.Module): + def __init__(self, state_size, action_size, hidden_size, activation): + super(QNetwork, self).__init__() + self.q_head = nn.Sequential( + nn.Linear(state_size, hidden_size), + instantiate(activation), + nn.Linear(hidden_size, hidden_size), + instantiate(activation), + nn.Linear(hidden_size, action_size) + ) + + def forward(self, state): + Qs = self.q_head(state) + return Qs + + +class DuelingQNetwork(nn.Module): + def __init__(self, state_size, action_size, hidden_size, activation): + super(DuelingQNetwork, self).__init__() + self.feature_layer = nn.Sequential( + nn.Linear(state_size, hidden_size), + instantiate(activation), + ) + self.value_head = nn.Sequential( + nn.Linear(hidden_size, hidden_size), + instantiate(activation), + nn.Linear(hidden_size, 1) + ) + self.advantage_head = nn.Sequential( + nn.Linear(hidden_size, hidden_size), + instantiate(activation), + nn.Linear(hidden_size, action_size) + ) + + def forward(self, state): + """ + Get the Q value of the current state and action using dueling network + """ + ############################ + # YOUR IMPLEMENTATION HERE # + + # using equation (7) on https://arxiv.org/pdf/1511.06581 + Qs=self.value_head(self.feature_layer(state))+self.advantage_head(self.feature_layer(state)) + ############################ + return Qs + + + \ No newline at end of file diff --git a/hw2/utils.py b/hw2/utils.py new file mode 100644 index 0000000..8dc529c --- /dev/null +++ b/hw2/utils.py @@ -0,0 +1,65 @@ +import os +import glob +import torch +import shutil +import random +import numpy as np +import gymnasium as gym +from gymnasium.spaces import Discrete, Box +from moviepy.editor import VideoFileClip, concatenate_videoclips + + +def moving_average(a, n): + """ + Return an array of the moving average of a with window size n + """ + if len(a) <= n: + return a + ret = np.cumsum(a, dtype=float) + ret[n:] = ret[n:] - ret[:-n] + return ret[n - 1:] / n + +def get_epsilon(step, eps_min, eps_max, eps_steps): + """ + Return the linearly descending epsilon of the current step for the epsilon-greedy policy. After eps_steps, epsilon will keep at eps_min + """ + ############################ + # YOUR IMPLEMENTATION HERE # + return max(eps_min, eps_max - (step / eps_steps) * (eps_max - eps_min)) + ############################ + +def merge_videos(video_dir): + """ + Merge videos in the video_dir into a single video + """ + videos = glob.glob(os.path.join(video_dir, "*.mp4")) + videos = sorted(videos, key=lambda x: int(x.split("-")[-1].split(".")[0])) + clip = concatenate_videoclips([VideoFileClip(video) for video in videos]) + clip.write_videofile(f"{video_dir}.mp4") + shutil.rmtree(video_dir) + + +def set_seed_everywhere(env: gym.Env, seed=0): + """ + Set seed for all randomness sources + """ + env.action_space.seed(seed) + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + + +def get_space_shape(space): + """ + Return the shape of the gym.Space object + """ + if isinstance(space, Discrete): + return space.n + elif isinstance(space, Box): + if len(space.shape) == 1: + return space.shape[0] + else: + return space.shape + else: + raise ValueError(f"Space not supported: {space}") diff --git a/result.aux b/result.aux new file mode 100644 index 0000000..7289211 --- /dev/null +++ b/result.aux @@ -0,0 +1,5 @@ +\relax +\newlabel{1}{{{1}}{1}{}{}{}} +\newlabel{2}{{{2}}{1}{}{}{}} +\newlabel{3}{{{3}}{1}{}{}{}} +\gdef \@abspage@last{4} diff --git a/result.fdb_latexmk b/result.fdb_latexmk new file mode 100644 index 0000000..9cf870e --- /dev/null +++ b/result.fdb_latexmk @@ -0,0 +1,80 @@ +# Fdb version 4 +["pdflatex"] 1760230164.51698 "d:/Documents/Nextcloud/Documents/Project WUSTL/Academic/2025_Fall/CSE5100/Homeworks/hw2/result.tex" "result.pdf" "result" 1760230165.61579 0 + "c:/texlive/2023/texmf-dist/fonts/map/fontname/texfonts.map" 1708990624 3524 cb3e574dea2d1052e39280babc910dc8 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex7.tfm" 1708988591 1004 54797486969f23fa377b128694d548df "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex8.tfm" 1708988591 988 bdf658c3bfc2d96d3c8b02cfc1c94c20 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm" 1708988591 916 f87d7c45f9c908e672703b83b72241a3 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam7.tfm" 1708988591 928 2dc8d444221b7a635bb58038579b861a "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm" 1708988591 908 2921f8a10601f252058503cc6570e581 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm7.tfm" 1708988591 940 228d6584342e91276bf566bcf9716b83 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm" 1708989536 1328 c834bbb027764024c09d3d2bf908b5f0 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmex10.tfm" 1708989536 992 662f679a0b3d2d53c1b94050fdaa3f50 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm" 1708989536 1528 abec98dbc43e172678c11b3b9031252a "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi6.tfm" 1708989536 1512 f21f83efb36853c0b70002322c1ab3ad "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi8.tfm" 1708989536 1520 eccf95517727cb11801f4f1aee3a21b4 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr10.tfm" 1708989536 1296 45809c5a464d5f32c8f98ba97c1bb47f "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr6.tfm" 1708989536 1300 b62933e007d01cfd073f79b963c01526 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr8.tfm" 1708989536 1292 21c1c5bfeaebccffdb478fd231a0997d "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm" 1708989536 1124 6c73e740cf17375f03eec0ee63599741 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy6.tfm" 1708989536 1116 933a60c408fc0a863a92debe84b2d294 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy8.tfm" 1708989536 1120 8b7d695260f3cff42e636090a8002094 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmti10.tfm" 1708989536 1480 aa8e34af0eb6a2941b776984cf1dfdc4 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/rsfs/rsfs10.tfm" 1708993366 688 37338d6ab346c2f1466b29e195316aa4 "" + "c:/texlive/2023/texmf-dist/fonts/tfm/public/rsfs/rsfs5.tfm" 1708993366 684 3a51bd4fd9600428d5264cf25f04bb9a "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmbx10.pfb" 1708988591 34811 78b52f49e893bcba91bd7581cdc144c0 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmex10.pfb" 1708988591 30251 6afa5cb1d0204815a708a080681d4674 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb" 1708988591 36299 5f9df58c2139e7edcf37c8fca4bd384d "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi8.pfb" 1708988591 35469 dcf3a5f2fc1862f5952e3ee5eb1d98c4 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb" 1708988591 35752 024fb6c41858982481f6968b5fc26508 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr8.pfb" 1708988591 32726 39f0f9e62e84beb801509898a605dbd5 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb" 1708988591 32569 5e5ddc8df908dea60932f3c484a54c0d "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy6.pfb" 1708988591 32587 65067f817f408bc71a7312f3d9828a9b "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy8.pfb" 1708988591 32626 5abc8bb2f28aa647d4c70f8ea38cc0d3 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmti10.pfb" 1708988591 37944 359e864bd06cde3b1cf57bb20757fb06 "" + "c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/symbols/msbm10.pfb" 1708988591 34694 870c211f62cb72718a00e353f14f254d "" + "c:/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii" 1708992232 71627 94eb9990bed73c364d7f53f960cc8c5b "" + "c:/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty" 1708991801 17865 1a9bd36b4f98178fa551aca822290953 "" + "c:/texlive/2023/texmf-dist/tex/latex/amscls/amsthm.sty" 1708988587 12594 0d51ac3a545aaaa555021326ff22a6cc "" + "c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty" 1708988591 5949 3f3fd50a8cc94c3d4cbf4fc66cd3df1c "" + "c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amssymb.sty" 1708988591 13829 94730e64147574077f8ecfea9bb69af4 "" + "c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd" 1708988592 961 6518c6525a34feb5e8250ffa91731cff "" + "c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd" 1708988592 961 d02606146ba5601b5645f987c92e6193 "" + "c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsbsy.sty" 1708988596 2222 499d61426192c39efd8f410ee1a52b9c "" + "c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsgen.sty" 1708988596 4173 82ac04dfb1256038fad068287fbb4fe6 "" + "c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsmath.sty" 1708988596 88371 d84032c0f422c3d1e282266c01bef237 "" + "c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsopn.sty" 1708988596 4474 b811654f4bf125f11506d13d13647efb "" + "c:/texlive/2023/texmf-dist/tex/latex/amsmath/amstext.sty" 1708988596 2444 0d0c1ee65478277e8015d65b86983da2 "" + "c:/texlive/2023/texmf-dist/tex/latex/base/article.cls" 1708991500 20144 147463a6a579f4597269ef9565205cfe "" + "c:/texlive/2023/texmf-dist/tex/latex/base/size11.clo" 1708991500 8464 59874a3b0776c73e2a138b025d8473dd "" + "c:/texlive/2023/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty" 1708990302 13886 d1306dcf79a944f6988e688c1785f9ce "" + "c:/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty" 1708990361 46845 3b58f70c6e861a13d927bff09d35ecbc "" + "c:/texlive/2023/texmf-dist/tex/latex/fancyhdr/fancyhdr.sty" 1708990446 18450 88279bf67c81e69f8e3f1c1bad1a26c5 "" + "c:/texlive/2023/texmf-dist/tex/latex/graphics-cfg/graphics.cfg" 1708990878 1224 978390e9c2234eab29404bc21b268d1e "" + "c:/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def" 1708990879 19448 1e988b341dda20961a6b931bcde55519 "" + "c:/texlive/2023/texmf-dist/tex/latex/graphics/graphics.sty" 1708990876 18387 8f900a490197ebaf93c02ae9476d4b09 "" + "c:/texlive/2023/texmf-dist/tex/latex/graphics/graphicx.sty" 1708990876 8010 a8d949cbdbc5c983593827c9eec252e1 "" + "c:/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty" 1708990876 2671 7e67d78d9b88c845599a85b2d41f2e39 "" + "c:/texlive/2023/texmf-dist/tex/latex/graphics/trig.sty" 1708990876 4023 293ea1c16429fc0c4cf605f4da1791a9 "" + "c:/texlive/2023/texmf-dist/tex/latex/jknapltx/mathrsfs.sty" 1708991316 300 12fa6f636b617656f2810ee82cb05015 "" + "c:/texlive/2023/texmf-dist/tex/latex/jknapltx/ursfs.fd" 1708991316 548 cc4e3557704bfed27c7002773fad6c90 "" + "c:/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty" 1708991458 22555 6d8e155cfef6d82c3d5c742fea7c992e "" + "c:/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty" 1708991460 13815 760b0c02f691ea230f5359c4e1de23a7 "" + "c:/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def" 1708991467 30006 3d512c0edd558928ddea1690180ef77e "" + "c:/texlive/2023/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg" 1708991573 678 4792914a8f45be57bb98413425e4c7af "" + "c:/texlive/2023/texmf-dist/tex/latex/mathtools/mathtools.sty" 1708991994 62269 5c1837a5bc5db4c0d255eedc225ca44b "" + "c:/texlive/2023/texmf-dist/tex/latex/mathtools/mhsetup.sty" 1708991994 5582 a43dedf8e5ec418356f1e9dfe5d29fc3 "" + "c:/texlive/2023/texmf-dist/tex/latex/parskip/parskip.sty" 1708992628 4288 94714aa7f535440f33181fec52a31963 "" + "c:/texlive/2023/texmf-dist/tex/latex/preprint/fullpage.sty" 1708992914 2789 05b418f78b224ec872f5b11081138605 "" + "c:/texlive/2023/texmf-dist/tex/latex/tools/calc.sty" 1708994243 10214 547fd4d29642cb7c80bf54b49d447f01 "" + "c:/texlive/2023/texmf-dist/web2c/texmf.cnf" 1708988443 41009 84b61f42d16d06bedb915f57aa2374cf "" + "c:/texlive/2023/texmf-var/fonts/map/pdftex/updmap/pdftex.map" 1708994999 5518052 de2a91c664d75f3971de4662dc6b5a65 "" + "c:/texlive/2023/texmf-var/web2c/pdftex/pdflatex.fmt" 1708995327 8220658 fb4d14532342a0ef5245dd396c4a1bd1 "" + "c:/texlive/2023/texmf.cnf" 1708994944 713 e69b156964470283e0530f5060668171 "" + "d:/Documents/Nextcloud/Documents/Project WUSTL/Academic/2025_Fall/CSE5100/Homeworks/hw2/result.tex" 1760230163 6257 29867be9781c52dc4faad49bb0cba6fa "" + "result.aux" 1760230165 119 495be67432001ea8f4f9fa642ad39ad3 "pdflatex" + "result.tex" 1760230163 6257 29867be9781c52dc4faad49bb0cba6fa "" + (generated) + "result.aux" + "result.log" + "result.pdf" + (rewritten before read) diff --git a/result.fls b/result.fls new file mode 100644 index 0000000..d6af5b2 --- /dev/null +++ b/result.fls @@ -0,0 +1,124 @@ +PWD d:/Documents/Nextcloud/Documents/Project WUSTL/Academic/2025_Fall/CSE5100/Homeworks/hw2 +INPUT c:/texlive/2023/texmf.cnf +INPUT c:/texlive/2023/texmf-dist/web2c/texmf.cnf +INPUT c:/texlive/2023/texmf-var/web2c/pdftex/pdflatex.fmt +INPUT d:/Documents/Nextcloud/Documents/Project WUSTL/Academic/2025_Fall/CSE5100/Homeworks/hw2/result.tex +OUTPUT result.log +INPUT c:/texlive/2023/texmf-dist/tex/latex/base/article.cls +INPUT c:/texlive/2023/texmf-dist/tex/latex/base/article.cls +INPUT c:/texlive/2023/texmf-dist/tex/latex/base/size11.clo +INPUT c:/texlive/2023/texmf-dist/tex/latex/base/size11.clo +INPUT c:/texlive/2023/texmf-dist/tex/latex/base/size11.clo +INPUT c:/texlive/2023/texmf-dist/fonts/map/fontname/texfonts.map +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr10.tfm +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsmath.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsmath.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsopn.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amstext.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amstext.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsgen.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsgen.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsbsy.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsbsy.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsopn.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amscls/amsthm.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amscls/amsthm.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amssymb.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amssymb.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/fancyhdr/fancyhdr.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/fancyhdr/fancyhdr.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/parskip/parskip.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/parskip/parskip.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty +INPUT c:/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty +INPUT c:/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/preprint/fullpage.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/preprint/fullpage.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/jknapltx/mathrsfs.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/jknapltx/mathrsfs.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/mathtools/mathtools.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/mathtools/mathtools.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/tools/calc.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/tools/calc.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/mathtools/mhsetup.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/mathtools/mhsetup.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def +INPUT c:/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def +INPUT ./result.aux +INPUT ./result.aux +INPUT result.aux +OUTPUT result.aux +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/graphicx.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/graphicx.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/graphics.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/graphics.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/trig.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics/trig.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def +INPUT c:/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def +INPUT c:/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii +INPUT c:/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii +INPUT c:/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii +INPUT c:/texlive/2023/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty +INPUT c:/texlive/2023/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +INPUT c:/texlive/2023/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +INPUT c:/texlive/2023/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmbx10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr8.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmr6.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi8.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmmi6.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy8.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmsy6.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmex10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex8.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/cmextra/cmex7.tfm +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msam7.tfm +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd +INPUT c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/amsfonts/symbols/msbm7.tfm +INPUT c:/texlive/2023/texmf-dist/tex/latex/jknapltx/ursfs.fd +INPUT c:/texlive/2023/texmf-dist/tex/latex/jknapltx/ursfs.fd +INPUT c:/texlive/2023/texmf-dist/tex/latex/jknapltx/ursfs.fd +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/rsfs/rsfs10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/rsfs/rsfs10.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/rsfs/rsfs5.tfm +INPUT c:/texlive/2023/texmf-dist/fonts/tfm/public/cm/cmti10.tfm +OUTPUT result.pdf +INPUT c:/texlive/2023/texmf-var/fonts/map/pdftex/updmap/pdftex.map +INPUT result.aux +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmbx10.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmex10.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi10.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi8.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmr8.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy6.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy8.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/cm/cmti10.pfb +INPUT c:/texlive/2023/texmf-dist/fonts/type1/public/amsfonts/symbols/msbm10.pfb diff --git a/result.log b/result.log new file mode 100644 index 0000000..c78390c --- /dev/null +++ b/result.log @@ -0,0 +1,260 @@ +This is pdfTeX, Version 3.141592653-2.6-1.40.25 (TeX Live 2023) (preloaded format=pdflatex 2024.2.26) 11 OCT 2025 19:49 +entering extended mode + restricted \write18 enabled. + file:line:error style messages enabled. + %&-line parsing enabled. +**"d:/Documents/Nextcloud/Documents/Project WUSTL/Academic/2025_Fall/CSE5100/Homeworks/hw2/result.tex" +(d:/Documents/Nextcloud/Documents/Project WUSTL/Academic/2025_Fall/CSE5100/Homeworks/hw2/result.tex +LaTeX2e <2023-11-01> patch level 1 +L3 programming layer <2024-02-20> +(c:/texlive/2023/texmf-dist/tex/latex/base/article.cls +Document Class: article 2023/05/17 v1.4n Standard LaTeX document class +(c:/texlive/2023/texmf-dist/tex/latex/base/size11.clo +File: size11.clo 2023/05/17 v1.4n Standard LaTeX file (size option) +) +\c@part=\count188 +\c@section=\count189 +\c@subsection=\count190 +\c@subsubsection=\count191 +\c@paragraph=\count192 +\c@subparagraph=\count193 +\c@figure=\count194 +\c@table=\count195 +\abovecaptionskip=\skip48 +\belowcaptionskip=\skip49 +\bibindent=\dimen140 +) (c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsmath.sty +Package: amsmath 2023/05/13 v2.17o AMS math features +\@mathmargin=\skip50 + +For additional information on amsmath, use the `?' option. +(c:/texlive/2023/texmf-dist/tex/latex/amsmath/amstext.sty +Package: amstext 2021/08/26 v2.01 AMS text + (c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsgen.sty +File: amsgen.sty 1999/11/30 v2.0 generic functions +\@emptytoks=\toks17 +\ex@=\dimen141 +)) (c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsbsy.sty +Package: amsbsy 1999/11/29 v1.2d Bold Symbols +\pmbraise@=\dimen142 +) (c:/texlive/2023/texmf-dist/tex/latex/amsmath/amsopn.sty +Package: amsopn 2022/04/08 v2.04 operator names +) +\inf@bad=\count196 +LaTeX Info: Redefining \frac on input line 234. +\uproot@=\count197 +\leftroot@=\count198 +LaTeX Info: Redefining \overline on input line 399. +LaTeX Info: Redefining \colon on input line 410. +\classnum@=\count199 +\DOTSCASE@=\count266 +LaTeX Info: Redefining \ldots on input line 496. +LaTeX Info: Redefining \dots on input line 499. +LaTeX Info: Redefining \cdots on input line 620. +\Mathstrutbox@=\box51 +\strutbox@=\box52 +LaTeX Info: Redefining \big on input line 722. +LaTeX Info: Redefining \Big on input line 723. +LaTeX Info: Redefining \bigg on input line 724. +LaTeX Info: Redefining \Bigg on input line 725. +\big@size=\dimen143 +LaTeX Font Info: Redeclaring font encoding OML on input line 743. +LaTeX Font Info: Redeclaring font encoding OMS on input line 744. +\macc@depth=\count267 +LaTeX Info: Redefining \bmod on input line 905. +LaTeX Info: Redefining \pmod on input line 910. +LaTeX Info: Redefining \smash on input line 940. +LaTeX Info: Redefining \relbar on input line 970. +LaTeX Info: Redefining \Relbar on input line 971. +\c@MaxMatrixCols=\count268 +\dotsspace@=\muskip16 +\c@parentequation=\count269 +\dspbrk@lvl=\count270 +\tag@help=\toks18 +\row@=\count271 +\column@=\count272 +\maxfields@=\count273 +\andhelp@=\toks19 +\eqnshift@=\dimen144 +\alignsep@=\dimen145 +\tagshift@=\dimen146 +\tagwidth@=\dimen147 +\totwidth@=\dimen148 +\lineht@=\dimen149 +\@envbody=\toks20 +\multlinegap=\skip51 +\multlinetaggap=\skip52 +\mathdisplay@stack=\toks21 +LaTeX Info: Redefining \[ on input line 2953. +LaTeX Info: Redefining \] on input line 2954. +) (c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amsfonts.sty +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Redeclaring math symbol \hbar on input line 98. +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +) (c:/texlive/2023/texmf-dist/tex/latex/amscls/amsthm.sty +Package: amsthm 2020/05/29 v2.20.6 +\thm@style=\toks22 +\thm@bodyfont=\toks23 +\thm@headfont=\toks24 +\thm@notefont=\toks25 +\thm@headpunct=\toks26 +\thm@preskip=\skip53 +\thm@postskip=\skip54 +\thm@headsep=\skip55 +\dth@everypar=\toks27 +) (c:/texlive/2023/texmf-dist/tex/latex/amsfonts/amssymb.sty +Package: amssymb 2013/01/14 v3.01 AMS font symbols +) (c:/texlive/2023/texmf-dist/tex/latex/fancyhdr/fancyhdr.sty +Package: fancyhdr 2022/11/09 v4.1 Extensive control of page headers and footers +\f@nch@headwidth=\skip56 +\f@nch@O@elh=\skip57 +\f@nch@O@erh=\skip58 +\f@nch@O@olh=\skip59 +\f@nch@O@orh=\skip60 +\f@nch@O@elf=\skip61 +\f@nch@O@erf=\skip62 +\f@nch@O@olf=\skip63 +\f@nch@O@orf=\skip64 +) (c:/texlive/2023/texmf-dist/tex/latex/parskip/parskip.sty +Package: parskip 2021-03-14 v2.0h non-zero parskip adjustments + (c:/texlive/2023/texmf-dist/tex/latex/kvoptions/kvoptions.sty +Package: kvoptions 2022-06-15 v3.15 Key value format for package options (HO) + (c:/texlive/2023/texmf-dist/tex/latex/graphics/keyval.sty +Package: keyval 2022/05/29 v1.15 key=value parser (DPC) +\KV@toks@=\toks28 +) (c:/texlive/2023/texmf-dist/tex/generic/ltxcmds/ltxcmds.sty +Package: ltxcmds 2023-12-04 v1.26 LaTeX kernel commands for general use (HO) +) (c:/texlive/2023/texmf-dist/tex/latex/kvsetkeys/kvsetkeys.sty +Package: kvsetkeys 2022-10-05 v1.19 Key value parser (HO) +)) (c:/texlive/2023/texmf-dist/tex/latex/etoolbox/etoolbox.sty +Package: etoolbox 2020/10/05 v2.5k e-TeX tools for LaTeX (JAW) +\etb@tempcnta=\count274 +)) (c:/texlive/2023/texmf-dist/tex/latex/preprint/fullpage.sty +Package: fullpage 1999/02/23 1.1 (PWD) +\FP@margin=\skip65 +) (c:/texlive/2023/texmf-dist/tex/latex/jknapltx/mathrsfs.sty +Package: mathrsfs 1996/01/01 Math RSFS package v1.0 (jk) +\symrsfs=\mathgroup6 +) (c:/texlive/2023/texmf-dist/tex/latex/mathtools/mathtools.sty +Package: mathtools 2022/06/29 v1.29 mathematical typesetting tools + (c:/texlive/2023/texmf-dist/tex/latex/tools/calc.sty +Package: calc 2023/07/08 v4.3 Infix arithmetic (KKT,FJ) +\calc@Acount=\count275 +\calc@Bcount=\count276 +\calc@Adimen=\dimen150 +\calc@Bdimen=\dimen151 +\calc@Askip=\skip66 +\calc@Bskip=\skip67 +LaTeX Info: Redefining \setlength on input line 80. +LaTeX Info: Redefining \addtolength on input line 81. +\calc@Ccount=\count277 +\calc@Cskip=\skip68 +) (c:/texlive/2023/texmf-dist/tex/latex/mathtools/mhsetup.sty +Package: mhsetup 2021/03/18 v1.4 programming setup (MH) +) +\g_MT_multlinerow_int=\count278 +\l_MT_multwidth_dim=\dimen152 +\origjot=\skip69 +\l_MT_shortvdotswithinadjustabove_dim=\dimen153 +\l_MT_shortvdotswithinadjustbelow_dim=\dimen154 +\l_MT_above_intertext_sep=\dimen155 +\l_MT_below_intertext_sep=\dimen156 +\l_MT_above_shortintertext_sep=\dimen157 +\l_MT_below_shortintertext_sep=\dimen158 +\xmathstrut@box=\box53 +\xmathstrut@dim=\dimen159 +) +\c@theorem=\count279 + (c:/texlive/2023/texmf-dist/tex/latex/l3backend/l3backend-pdftex.def +File: l3backend-pdftex.def 2024-02-20 L3 backend support: PDF output (pdfTeX) +\l__color_backend_stack_int=\count280 +\l__pdf_internal_box=\box54 +) (./result.aux) +\openout1 = `result.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 77. +LaTeX Font Info: ... okay on input line 77. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 77. +LaTeX Font Info: ... okay on input line 77. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 77. +LaTeX Font Info: ... okay on input line 77. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 77. +LaTeX Font Info: ... okay on input line 77. +LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 77. +LaTeX Font Info: ... okay on input line 77. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 77. +LaTeX Font Info: ... okay on input line 77. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 77. +LaTeX Font Info: ... okay on input line 77. + (c:/texlive/2023/texmf-dist/tex/latex/graphics/graphicx.sty +Package: graphicx 2021/09/16 v1.2d Enhanced LaTeX Graphics (DPC,SPQR) + (c:/texlive/2023/texmf-dist/tex/latex/graphics/graphics.sty +Package: graphics 2022/03/10 v1.4e Standard LaTeX Graphics (DPC,SPQR) + (c:/texlive/2023/texmf-dist/tex/latex/graphics/trig.sty +Package: trig 2021/08/11 v1.11 sin cos tan (DPC) +) (c:/texlive/2023/texmf-dist/tex/latex/graphics-cfg/graphics.cfg +File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration +) +Package graphics Info: Driver file: pdftex.def on input line 107. + (c:/texlive/2023/texmf-dist/tex/latex/graphics-def/pdftex.def +File: pdftex.def 2022/09/22 v1.2b Graphics/color driver for pdftex + (c:/texlive/2023/texmf-dist/tex/context/base/mkii/supp-pdf.mkii +[Loading MPS to PDF converter (version 2006.09.02).] +\scratchcounter=\count281 +\scratchdimen=\dimen160 +\scratchbox=\box55 +\nofMPsegments=\count282 +\nofMParguments=\count283 +\everyMPshowfont=\toks29 +\MPscratchCnt=\count284 +\MPscratchDim=\dimen161 +\MPnumerator=\count285 +\makeMPintoPDFobject=\count286 +\everyMPtoPDFconversion=\toks30 +))) (c:/texlive/2023/texmf-dist/tex/latex/epstopdf-pkg/epstopdf-base.sty +Package: epstopdf-base 2020-01-24 v2.11 Base part for package epstopdf +Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 485. + (c:/texlive/2023/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +File: epstopdf-sys.cfg 2010/07/13 v1.3 Configuration of (r)epstopdf for TeX Live +)) +\Gin@req@height=\dimen162 +\Gin@req@width=\dimen163 +) +LaTeX Font Info: Trying to load font information for U+msa on input line 88. + (c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsa.fd +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) +LaTeX Font Info: Trying to load font information for U+msb on input line 88. + (c:/texlive/2023/texmf-dist/tex/latex/amsfonts/umsb.fd +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) +LaTeX Font Info: Trying to load font information for U+rsfs on input line 88. + (c:/texlive/2023/texmf-dist/tex/latex/jknapltx/ursfs.fd +File: ursfs.fd 1998/03/24 rsfs font definition file (jk) +) [1{c:/texlive/2023/texmf-var/fonts/map/pdftex/updmap/pdftex.map} + +] [2] [3] [4] (./result.aux) + *********** +LaTeX2e <2023-11-01> patch level 1 +L3 programming layer <2024-02-20> + *********** + ) +Here is how much of TeX's memory you used: + 4245 strings out of 474137 + 64387 string characters out of 5748517 + 1938190 words of memory out of 5000000 + 26537 multiletter control sequences out of 15000+600000 + 563865 words of font info for 59 fonts, out of 8000000 for 9000 + 1141 hyphenation exceptions out of 8191 + 65i,11n,72p,713b,463s stack positions out of 10000i,1000n,20000p,200000b,200000s + +Output written on result.pdf (4 pages, 122012 bytes). +PDF statistics: + 72 PDF objects out of 1000 (max. 8388607) + 43 compressed objects within 1 object stream + 0 named destinations out of 1000 (max. 500000) + 1 words of extra memory for PDF output out of 10000 (max. 10000000) + diff --git a/result.pdf b/result.pdf new file mode 100644 index 0000000..0bf141e Binary files /dev/null and b/result.pdf differ diff --git a/result.synctex.gz b/result.synctex.gz new file mode 100644 index 0000000..a9f8d3d Binary files /dev/null and b/result.synctex.gz differ diff --git a/result.tex b/result.tex new file mode 100644 index 0000000..63793d6 --- /dev/null +++ b/result.tex @@ -0,0 +1,156 @@ +\documentclass[11pt]{article} +\usepackage{amsmath, amsfonts, amsthm} +\usepackage{amssymb} +\usepackage{fancyhdr,parskip} +\usepackage{fullpage} +\usepackage{mathrsfs} +\usepackage{mathtools} + +%% +%% Stuff above here is packages that will be used to compile your document. +%% If you've used unusual LaTeX features, you may have to install extra packages by adding them to this list. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +\setlength{\headheight}{15.2pt} +\setlength{\headsep}{20pt} +\pagestyle{fancyplain} + +%% +%% Stuff above here is layout and formatting. If you've never used LaTeX before, you probably don't need to change any of it. +%% Later, you can learn how it all works and adjust it to your liking, or write your own formatting code. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%%%%%%%%%%%%%%%%%%%%%% +% These commands create theorem-like environments. +\newtheorem{theorem}{Theorem} +\newtheorem{lemma}[theorem]{Lemma} +\newtheorem{corollary}[theorem]{Corollary} +\newtheorem{prop}[theorem]{Proposition} +\newtheorem{defn}[theorem]{Definition} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% This section contains some useful macros that will save you time typing. +%% + +% Using \displaystyle (or \ds) in a block of math has a number of effects, but most notably, it makes your fractions come out bigger. +\newcommand{\ds}{\displaystyle} + +% These lines are for displaying integrals; typing \dx will make the dx at the end of the integral look better. +\newcommand{\is}{\hspace{2pt}} +\newcommand{\dx}{\is dx} + +% These commands produce the fancy Z (for the integers) and other letters conveniently. +\newcommand{\Z}{\mathbb{Z}} +\newcommand{\Q}{\mathbb{Q}} +\newcommand{\R}{\mathbb{R}} +\newcommand{\C}{\mathbb{C}} +\newcommand{\F}{\mathbb{F}} +\newcommand{\T}{\mathcal{T}} +\newcommand{\B}{\mathcal{B}} + +% for fancy empty set char +\renewcommand{\emptyset}{\varnothing} + +% customized commands for future assignements +\newcommand{\imply}{\Rightarrow} +\def\P{\mathscr{P}} +\def\L{\mathscr{L}} +\def\M{\mathscr{M}} +\DeclarePairedDelimiterX{\inp}[2]{\langle}{\rangle}{#1, #2} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% This is the header. It will appear on every page, and it's a good place to put your name, the assignment title, and stuff like that. +%% I usually leave the center header blank to avoid clutter. +%% + +\fancyhead[L]{\textbf{CSE5100 Homework 2}} +\fancyhead[C]{\empty} +\fancyhead[R]{Zheyuan Wu} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +\begin{document} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Actual math starts here! + + +% Use an enumerated list to write up problems. First we begin a list. + +\begin{enumerate} + +\item[1.] \textbf{Answer questions in Section 3} Due to the state space complexity of some visual input environments, we may represent Q-functions using a class of parameterized function approximators $\mathcal{Q}=\{Q_w\mid w\in \R^p\}$, where $p$ is the number of parameters. Remember that in the \textit{tabular setting} given a 4-tuple of sampled experience $(s,a,r,s')$, the vanilla Q-learning update is + +\[ +Q(s,a)\coloneqq Q(s,a)+\alpha\left(r+\gamma\max_{a'\in A} Q(s',a')-Q(s,a)\right),\tag{1}\label{1} +\] + +where $\alpha\in \R$ is the learning rate. In the \textit{function approximation setting}, the update is similar: + +\[ +w\coloneqq w+\alpha\left(r+\gamma\max_{a'\in A} Q_w(s',a')-Q_w(s,a)\right)\nabla_w Q_w(s,a).\tag{2}\label{2} +\] + +Q-learning can seem as a pseudo stochastic gradient descent step on + +\[ +\ell(w)=\mathbb{E}_{s,a,r,s'}\left(r+\gamma \max_{a'\in A} Q_w(s',a')-Q_w(s,a)\right)^2.\tag{3}\label{3} +\] + +where the dependency of $\max_{a'\in A} Q_w(s',a')$ on $w$ is ignored, i.e., it is treated as a fixed target. + +\begin{enumerate} + \item [1.] [\textbf{10pt}] Show that the update \ref{1} and update \ref{2} are the same when the functions in $\mathcal{Q}$ are of the form $Q_w(s,a)=w^T\phi(s,a)$, with $w\in \R^{|S||A|}$ and $\phi:S\times A\to \R^{|S||A|}$, where the feature function $\phi$ is of the form $\phi(s,a)_{s',a'}=\mathbb{I}[s'=s,a'=a]$, where $\mathbb{I}$ denotes the indicator function which evaluates to $1$ if the condition evaluates to true and vice versa. Note that the coordinates in the vector space $\R^{|S||A|}$ can be seen as being indexed by pairs $(s',a')$, where $s'\in S$, $a'\in A$. + + \begin{proof} + When the functions in $\mathcal{Q}$ are of the form $Q_w(s,a)=w^T\phi(s,a)$, with $w\in \R^{|S||A|}$ and $\phi:S\times A\to \R^{|S||A|}$, then it is linear. + + \[ + \begin{aligned} + Q(s,a)&= Q(s,a)+\alpha\left(r+\gamma\max_{a'\in A} Q(s',a')-Q(s,a)\right)\\ + w^T\phi(s,a)&= w^T\phi(s,a)+\alpha\left(r+\gamma\max_{a'\in A} Q(s',a')-Q(s,a)\right)\\ + w&= w+\alpha\left(r+\gamma\max_{a'\in A} Q(s',a')-Q(s,a)\right)\nabla_w Q_w(s,a) + \end{aligned} + \] + \end{proof} + + + \item [2.] [\textbf{10pt}] What is the deadly triad in the reinforcement learning? What are the main challenges of using deep learning for function approximation with Q-learning? How does Deep Q-Learning method overcome these challenges? + + The deadly triad in the reinforcement learning are + + \begin{enumerate} + \item Bootstraping + \item Function approximation + \item Off-policy + \end{enumerate} + + The Deep Q-Learning method overcome the instability caused by the deadly triad interact with statistical estimation issues induced by the bootstrap method used by boostrapping on a separate network and by reducing the overestimation bias. (Use double Q-learning to reduce the overestimation bias.) + + \item [3.] [\textbf{10pt}] Explain how double Q-learning helps with the maximization bias in Q-learning. + + The double Q-learning decouple the action selection and evaluation of action to separate networks. + +\end{enumerate} + +\newpage + +\item [2.] \textbf{The auto-generated results figure} along with a brief description about what has the figures shown. + +\newpage + +\item [3.] \textbf{Any other findings} + +\end{enumerate} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Actual math ends here. Don't put any content below the \end{document} line. +%% + +\end{document} diff --git a/runs/2025-10-11/20-24-05_/.hydra/config.yaml b/runs/2025-10-11/20-24-05_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-24-05_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-24-05_/.hydra/hydra.yaml b/runs/2025-10-11/20-24-05_/.hydra/hydra.yaml new file mode 100644 index 0000000..ac56f45 --- /dev/null +++ b/runs/2025-10-11/20-24-05_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-24-05_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-24-05_/.hydra/overrides.yaml b/runs/2025-10-11/20-24-05_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-24-05_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-24-05_/main.log b/runs/2025-10-11/20-24-05_/main.log new file mode 100644 index 0000000..ef38529 --- /dev/null +++ b/runs/2025-10-11/20-24-05_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 20:24:05,493][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:173: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_61 sm_70 sm_75 sm_80 sm_86 compute_37. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn(incompatible_device_warn.format(device_name, capability, " ".join(arch_list), device_name)) + diff --git a/runs/2025-10-11/20-31-16_/.hydra/config.yaml b/runs/2025-10-11/20-31-16_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-31-16_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-31-16_/.hydra/hydra.yaml b/runs/2025-10-11/20-31-16_/.hydra/hydra.yaml new file mode 100644 index 0000000..88b0f04 --- /dev/null +++ b/runs/2025-10-11/20-31-16_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-31-16_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-31-16_/.hydra/overrides.yaml b/runs/2025-10-11/20-31-16_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-31-16_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-31-16_/main.log b/runs/2025-10-11/20-31-16_/main.log new file mode 100644 index 0000000..54adf50 --- /dev/null +++ b/runs/2025-10-11/20-31-16_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 20:31:16,113][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:173: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_61 sm_70 sm_75 sm_80 sm_86 compute_37. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn(incompatible_device_warn.format(device_name, capability, " ".join(arch_list), device_name)) + diff --git a/runs/2025-10-11/20-35-30_/.hydra/config.yaml b/runs/2025-10-11/20-35-30_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-35-30_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-35-30_/.hydra/hydra.yaml b/runs/2025-10-11/20-35-30_/.hydra/hydra.yaml new file mode 100644 index 0000000..ff5a31f --- /dev/null +++ b/runs/2025-10-11/20-35-30_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-35-30_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-35-30_/.hydra/overrides.yaml b/runs/2025-10-11/20-35-30_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-35-30_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-35-30_/main.log b/runs/2025-10-11/20-35-30_/main.log new file mode 100644 index 0000000..2b7fccd --- /dev/null +++ b/runs/2025-10-11/20-35-30_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 20:35:30,859][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:173: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_61 sm_70 sm_75 sm_80 sm_86 compute_37. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn(incompatible_device_warn.format(device_name, capability, " ".join(arch_list), device_name)) + diff --git a/runs/2025-10-11/20-39-45_/.hydra/config.yaml b/runs/2025-10-11/20-39-45_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-39-45_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-39-45_/.hydra/hydra.yaml b/runs/2025-10-11/20-39-45_/.hydra/hydra.yaml new file mode 100644 index 0000000..287bd6e --- /dev/null +++ b/runs/2025-10-11/20-39-45_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-39-45_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-39-45_/.hydra/overrides.yaml b/runs/2025-10-11/20-39-45_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-39-45_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-39-45_/main.log b/runs/2025-10-11/20-39-45_/main.log new file mode 100644 index 0000000..0ca4774 --- /dev/null +++ b/runs/2025-10-11/20-39-45_/main.log @@ -0,0 +1,22 @@ +[2025-10-11 20:39:45,474][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:283: UserWarning: + Found GPU0 NVIDIA GeForce RTX 5090 which is of cuda capability 12.0. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (6.1) - (9.0) + + warnings.warn( + +[2025-10-11 20:39:45,474][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:304: UserWarning: + Please install PyTorch with a following CUDA + configurations: 12.8 12.9 following instructions at + https://pytorch.org/get-started/locally/ + + warnings.warn(matched_cuda_warn.format(matched_arches)) + +[2025-10-11 20:39:45,476][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:326: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_61 sm_70 sm_75 sm_80 sm_86 sm_90. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn( + +[2025-10-11 20:39:47,115][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/20-41-09_/.hydra/config.yaml b/runs/2025-10-11/20-41-09_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-41-09_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-41-09_/.hydra/hydra.yaml b/runs/2025-10-11/20-41-09_/.hydra/hydra.yaml new file mode 100644 index 0000000..157b81e --- /dev/null +++ b/runs/2025-10-11/20-41-09_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-41-09_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-41-09_/.hydra/overrides.yaml b/runs/2025-10-11/20-41-09_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-41-09_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-41-09_/main.log b/runs/2025-10-11/20-41-09_/main.log new file mode 100644 index 0000000..f285555 --- /dev/null +++ b/runs/2025-10-11/20-41-09_/main.log @@ -0,0 +1,22 @@ +[2025-10-11 20:41:09,978][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:283: UserWarning: + Found GPU0 NVIDIA GeForce RTX 5090 which is of cuda capability 12.0. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (6.1) - (9.0) + + warnings.warn( + +[2025-10-11 20:41:09,979][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:304: UserWarning: + Please install PyTorch with a following CUDA + configurations: 12.8 12.9 following instructions at + https://pytorch.org/get-started/locally/ + + warnings.warn(matched_cuda_warn.format(matched_arches)) + +[2025-10-11 20:41:09,979][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:326: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_61 sm_70 sm_75 sm_80 sm_86 sm_90. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn( + +[2025-10-11 20:41:11,670][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/20-42-26_/.hydra/config.yaml b/runs/2025-10-11/20-42-26_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-42-26_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-42-26_/.hydra/hydra.yaml b/runs/2025-10-11/20-42-26_/.hydra/hydra.yaml new file mode 100644 index 0000000..fc366af --- /dev/null +++ b/runs/2025-10-11/20-42-26_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-42-26_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-42-26_/.hydra/overrides.yaml b/runs/2025-10-11/20-42-26_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-42-26_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-42-26_/main.log b/runs/2025-10-11/20-42-26_/main.log new file mode 100644 index 0000000..4818070 --- /dev/null +++ b/runs/2025-10-11/20-42-26_/main.log @@ -0,0 +1,22 @@ +[2025-10-11 20:42:26,843][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:283: UserWarning: + Found GPU0 NVIDIA GeForce RTX 5090 which is of cuda capability 12.0. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (6.1) - (9.0) + + warnings.warn( + +[2025-10-11 20:42:26,844][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:304: UserWarning: + Please install PyTorch with a following CUDA + configurations: 12.8 12.9 following instructions at + https://pytorch.org/get-started/locally/ + + warnings.warn(matched_cuda_warn.format(matched_arches)) + +[2025-10-11 20:42:26,846][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:326: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_61 sm_70 sm_75 sm_80 sm_86 sm_90. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn( + +[2025-10-11 20:42:28,580][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/20-45-12_/.hydra/config.yaml b/runs/2025-10-11/20-45-12_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-45-12_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-45-12_/.hydra/hydra.yaml b/runs/2025-10-11/20-45-12_/.hydra/hydra.yaml new file mode 100644 index 0000000..c8ac0af --- /dev/null +++ b/runs/2025-10-11/20-45-12_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-45-12_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-45-12_/.hydra/overrides.yaml b/runs/2025-10-11/20-45-12_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-45-12_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-45-12_/main.log b/runs/2025-10-11/20-45-12_/main.log new file mode 100644 index 0000000..af753e6 --- /dev/null +++ b/runs/2025-10-11/20-45-12_/main.log @@ -0,0 +1,22 @@ +[2025-10-11 20:45:12,694][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:283: UserWarning: + Found GPU0 NVIDIA GeForce RTX 5090 which is of cuda capability 12.0. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (6.1) - (9.0) + + warnings.warn( + +[2025-10-11 20:45:12,694][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:304: UserWarning: + Please install PyTorch with a following CUDA + configurations: 12.8 12.9 following instructions at + https://pytorch.org/get-started/locally/ + + warnings.warn(matched_cuda_warn.format(matched_arches)) + +[2025-10-11 20:45:12,696][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:326: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_61 sm_70 sm_75 sm_80 sm_86 sm_90. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn( + +[2025-10-11 20:45:14,422][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/20-45-23_/.hydra/config.yaml b/runs/2025-10-11/20-45-23_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-45-23_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-45-23_/.hydra/hydra.yaml b/runs/2025-10-11/20-45-23_/.hydra/hydra.yaml new file mode 100644 index 0000000..74e4f8e --- /dev/null +++ b/runs/2025-10-11/20-45-23_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-45-23_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-45-23_/.hydra/overrides.yaml b/runs/2025-10-11/20-45-23_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-45-23_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-45-23_/main.log b/runs/2025-10-11/20-45-23_/main.log new file mode 100644 index 0000000..fd5b219 --- /dev/null +++ b/runs/2025-10-11/20-45-23_/main.log @@ -0,0 +1,22 @@ +[2025-10-11 20:45:23,927][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:283: UserWarning: + Found GPU0 NVIDIA GeForce RTX 5090 which is of cuda capability 12.0. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (6.1) - (9.0) + + warnings.warn( + +[2025-10-11 20:45:23,928][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:304: UserWarning: + Please install PyTorch with a following CUDA + configurations: 12.8 12.9 following instructions at + https://pytorch.org/get-started/locally/ + + warnings.warn(matched_cuda_warn.format(matched_arches)) + +[2025-10-11 20:45:23,930][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:326: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_61 sm_70 sm_75 sm_80 sm_86 sm_90. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn( + +[2025-10-11 20:45:25,714][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/20-55-32_/.hydra/config.yaml b/runs/2025-10-11/20-55-32_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-55-32_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-55-32_/.hydra/hydra.yaml b/runs/2025-10-11/20-55-32_/.hydra/hydra.yaml new file mode 100644 index 0000000..5f75470 --- /dev/null +++ b/runs/2025-10-11/20-55-32_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-55-32_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-55-32_/.hydra/overrides.yaml b/runs/2025-10-11/20-55-32_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-55-32_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-55-32_/main.log b/runs/2025-10-11/20-55-32_/main.log new file mode 100644 index 0000000..75da3b7 --- /dev/null +++ b/runs/2025-10-11/20-55-32_/main.log @@ -0,0 +1,22 @@ +[2025-10-11 20:55:32,238][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:283: UserWarning: + Found GPU0 NVIDIA GeForce RTX 5090 which is of cuda capability 12.0. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (6.1) - (9.0) + + warnings.warn( + +[2025-10-11 20:55:32,238][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:304: UserWarning: + Please install PyTorch with a following CUDA + configurations: 12.8 12.9 following instructions at + https://pytorch.org/get-started/locally/ + + warnings.warn(matched_cuda_warn.format(matched_arches)) + +[2025-10-11 20:55:32,240][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\torch\cuda\__init__.py:326: UserWarning: +NVIDIA GeForce RTX 5090 with CUDA capability sm_120 is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities sm_61 sm_70 sm_75 sm_80 sm_86 sm_90. +If you want to use the NVIDIA GeForce RTX 5090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ + + warnings.warn( + +[2025-10-11 20:55:33,876][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/20-59-59_/.hydra/config.yaml b/runs/2025-10-11/20-59-59_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/20-59-59_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/20-59-59_/.hydra/hydra.yaml b/runs/2025-10-11/20-59-59_/.hydra/hydra.yaml new file mode 100644 index 0000000..3fb40dc --- /dev/null +++ b/runs/2025-10-11/20-59-59_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\20-59-59_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/20-59-59_/.hydra/overrides.yaml b/runs/2025-10-11/20-59-59_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/20-59-59_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/20-59-59_/main.log b/runs/2025-10-11/20-59-59_/main.log new file mode 100644 index 0000000..96c6669 --- /dev/null +++ b/runs/2025-10-11/20-59-59_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:00:01,190][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-00-50_/.hydra/config.yaml b/runs/2025-10-11/21-00-50_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-00-50_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-00-50_/.hydra/hydra.yaml b/runs/2025-10-11/21-00-50_/.hydra/hydra.yaml new file mode 100644 index 0000000..b63eee3 --- /dev/null +++ b/runs/2025-10-11/21-00-50_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-00-50_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-00-50_/.hydra/overrides.yaml b/runs/2025-10-11/21-00-50_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-00-50_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-00-50_/main.log b/runs/2025-10-11/21-00-50_/main.log new file mode 100644 index 0000000..e26c62b --- /dev/null +++ b/runs/2025-10-11/21-00-50_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:00:52,388][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-03-24_/.hydra/config.yaml b/runs/2025-10-11/21-03-24_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-03-24_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-03-24_/.hydra/hydra.yaml b/runs/2025-10-11/21-03-24_/.hydra/hydra.yaml new file mode 100644 index 0000000..afed174 --- /dev/null +++ b/runs/2025-10-11/21-03-24_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-03-24_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-03-24_/.hydra/overrides.yaml b/runs/2025-10-11/21-03-24_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-03-24_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-03-24_/main.log b/runs/2025-10-11/21-03-24_/main.log new file mode 100644 index 0000000..ca4306d --- /dev/null +++ b/runs/2025-10-11/21-03-24_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:03:26,154][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-03-35_/.hydra/config.yaml b/runs/2025-10-11/21-03-35_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-03-35_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-03-35_/.hydra/hydra.yaml b/runs/2025-10-11/21-03-35_/.hydra/hydra.yaml new file mode 100644 index 0000000..03c0727 --- /dev/null +++ b/runs/2025-10-11/21-03-35_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-03-35_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-03-35_/.hydra/overrides.yaml b/runs/2025-10-11/21-03-35_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-03-35_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-03-35_/main.log b/runs/2025-10-11/21-03-35_/main.log new file mode 100644 index 0000000..399fd6b --- /dev/null +++ b/runs/2025-10-11/21-03-35_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:03:36,838][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-05-11_/.hydra/config.yaml b/runs/2025-10-11/21-05-11_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-05-11_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-05-11_/.hydra/hydra.yaml b/runs/2025-10-11/21-05-11_/.hydra/hydra.yaml new file mode 100644 index 0000000..defd09b --- /dev/null +++ b/runs/2025-10-11/21-05-11_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-05-11_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-05-11_/.hydra/overrides.yaml b/runs/2025-10-11/21-05-11_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-05-11_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-05-11_/main.log b/runs/2025-10-11/21-05-11_/main.log new file mode 100644 index 0000000..95dd74b --- /dev/null +++ b/runs/2025-10-11/21-05-11_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:05:12,880][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-07-21_/.hydra/config.yaml b/runs/2025-10-11/21-07-21_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-07-21_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-07-21_/.hydra/hydra.yaml b/runs/2025-10-11/21-07-21_/.hydra/hydra.yaml new file mode 100644 index 0000000..b849f22 --- /dev/null +++ b/runs/2025-10-11/21-07-21_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-07-21_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-07-21_/.hydra/overrides.yaml b/runs/2025-10-11/21-07-21_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-07-21_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-07-21_/main.log b/runs/2025-10-11/21-07-21_/main.log new file mode 100644 index 0000000..03c78b3 --- /dev/null +++ b/runs/2025-10-11/21-07-21_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:07:22,911][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-07-44_/.hydra/config.yaml b/runs/2025-10-11/21-07-44_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-07-44_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-07-44_/.hydra/hydra.yaml b/runs/2025-10-11/21-07-44_/.hydra/hydra.yaml new file mode 100644 index 0000000..6ccb43d --- /dev/null +++ b/runs/2025-10-11/21-07-44_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-07-44_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-07-44_/.hydra/overrides.yaml b/runs/2025-10-11/21-07-44_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-07-44_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-07-44_/main.log b/runs/2025-10-11/21-07-44_/main.log new file mode 100644 index 0000000..2432b8e --- /dev/null +++ b/runs/2025-10-11/21-07-44_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:07:45,823][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-08-54_/.hydra/config.yaml b/runs/2025-10-11/21-08-54_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-08-54_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-08-54_/.hydra/hydra.yaml b/runs/2025-10-11/21-08-54_/.hydra/hydra.yaml new file mode 100644 index 0000000..8bd3933 --- /dev/null +++ b/runs/2025-10-11/21-08-54_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-08-54_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-08-54_/.hydra/overrides.yaml b/runs/2025-10-11/21-08-54_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-08-54_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-08-54_/main.log b/runs/2025-10-11/21-08-54_/main.log new file mode 100644 index 0000000..8fa2d6a --- /dev/null +++ b/runs/2025-10-11/21-08-54_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:08:56,669][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-09-36_/.hydra/config.yaml b/runs/2025-10-11/21-09-36_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-09-36_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-09-36_/.hydra/hydra.yaml b/runs/2025-10-11/21-09-36_/.hydra/hydra.yaml new file mode 100644 index 0000000..77d6327 --- /dev/null +++ b/runs/2025-10-11/21-09-36_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-09-36_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-09-36_/.hydra/overrides.yaml b/runs/2025-10-11/21-09-36_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-09-36_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-09-36_/main.log b/runs/2025-10-11/21-09-36_/main.log new file mode 100644 index 0000000..94c0da9 --- /dev/null +++ b/runs/2025-10-11/21-09-36_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:09:38,404][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-10-00_/.hydra/config.yaml b/runs/2025-10-11/21-10-00_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-10-00_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-10-00_/.hydra/hydra.yaml b/runs/2025-10-11/21-10-00_/.hydra/hydra.yaml new file mode 100644 index 0000000..c284d4d --- /dev/null +++ b/runs/2025-10-11/21-10-00_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-10-00_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-10-00_/.hydra/overrides.yaml b/runs/2025-10-11/21-10-00_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-10-00_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-10-00_/main.log b/runs/2025-10-11/21-10-00_/main.log new file mode 100644 index 0000000..021606e --- /dev/null +++ b/runs/2025-10-11/21-10-00_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:10:02,340][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-15-36_/.hydra/config.yaml b/runs/2025-10-11/21-15-36_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-15-36_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-15-36_/.hydra/hydra.yaml b/runs/2025-10-11/21-15-36_/.hydra/hydra.yaml new file mode 100644 index 0000000..05efbdb --- /dev/null +++ b/runs/2025-10-11/21-15-36_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-15-36_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-15-36_/.hydra/overrides.yaml b/runs/2025-10-11/21-15-36_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-15-36_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-15-36_/main.log b/runs/2025-10-11/21-15-36_/main.log new file mode 100644 index 0000000..c127990 --- /dev/null +++ b/runs/2025-10-11/21-15-36_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:15:37,961][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-16-27_/.hydra/config.yaml b/runs/2025-10-11/21-16-27_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-16-27_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-16-27_/.hydra/hydra.yaml b/runs/2025-10-11/21-16-27_/.hydra/hydra.yaml new file mode 100644 index 0000000..db7139b --- /dev/null +++ b/runs/2025-10-11/21-16-27_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-16-27_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-16-27_/.hydra/overrides.yaml b/runs/2025-10-11/21-16-27_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-16-27_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-16-27_/main.log b/runs/2025-10-11/21-16-27_/main.log new file mode 100644 index 0000000..60c5f0d --- /dev/null +++ b/runs/2025-10-11/21-16-27_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:16:28,918][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-16-42_/.hydra/config.yaml b/runs/2025-10-11/21-16-42_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-16-42_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-16-42_/.hydra/hydra.yaml b/runs/2025-10-11/21-16-42_/.hydra/hydra.yaml new file mode 100644 index 0000000..5a60654 --- /dev/null +++ b/runs/2025-10-11/21-16-42_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-16-42_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-16-42_/.hydra/overrides.yaml b/runs/2025-10-11/21-16-42_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-16-42_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-16-42_/main.log b/runs/2025-10-11/21-16-42_/main.log new file mode 100644 index 0000000..1913fcf --- /dev/null +++ b/runs/2025-10-11/21-16-42_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:16:44,069][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-17-17_/.hydra/config.yaml b/runs/2025-10-11/21-17-17_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-17-17_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-17-17_/.hydra/hydra.yaml b/runs/2025-10-11/21-17-17_/.hydra/hydra.yaml new file mode 100644 index 0000000..cee38c1 --- /dev/null +++ b/runs/2025-10-11/21-17-17_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-17-17_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-17-17_/.hydra/overrides.yaml b/runs/2025-10-11/21-17-17_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-17-17_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-17-17_/main.log b/runs/2025-10-11/21-17-17_/main.log new file mode 100644 index 0000000..f29314c --- /dev/null +++ b/runs/2025-10-11/21-17-17_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:17:19,615][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-17-42_/.hydra/config.yaml b/runs/2025-10-11/21-17-42_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-17-42_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-17-42_/.hydra/hydra.yaml b/runs/2025-10-11/21-17-42_/.hydra/hydra.yaml new file mode 100644 index 0000000..93edc09 --- /dev/null +++ b/runs/2025-10-11/21-17-42_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-17-42_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-17-42_/.hydra/overrides.yaml b/runs/2025-10-11/21-17-42_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-17-42_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-17-42_/main.log b/runs/2025-10-11/21-17-42_/main.log new file mode 100644 index 0000000..c575bb3 --- /dev/null +++ b/runs/2025-10-11/21-17-42_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:17:43,810][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-18-30_/.hydra/config.yaml b/runs/2025-10-11/21-18-30_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-18-30_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-18-30_/.hydra/hydra.yaml b/runs/2025-10-11/21-18-30_/.hydra/hydra.yaml new file mode 100644 index 0000000..75887f2 --- /dev/null +++ b/runs/2025-10-11/21-18-30_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-18-30_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-18-30_/.hydra/overrides.yaml b/runs/2025-10-11/21-18-30_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-18-30_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-18-30_/main.log b/runs/2025-10-11/21-18-30_/main.log new file mode 100644 index 0000000..103e580 --- /dev/null +++ b/runs/2025-10-11/21-18-30_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:18:32,255][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-35-19_/.hydra/config.yaml b/runs/2025-10-11/21-35-19_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-35-19_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-35-19_/.hydra/hydra.yaml b/runs/2025-10-11/21-35-19_/.hydra/hydra.yaml new file mode 100644 index 0000000..96e0b24 --- /dev/null +++ b/runs/2025-10-11/21-35-19_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-35-19_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-35-19_/.hydra/overrides.yaml b/runs/2025-10-11/21-35-19_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-35-19_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-35-19_/main.log b/runs/2025-10-11/21-35-19_/main.log new file mode 100644 index 0000000..58d76c3 --- /dev/null +++ b/runs/2025-10-11/21-35-19_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:35:21,306][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-36-31_/.hydra/config.yaml b/runs/2025-10-11/21-36-31_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-36-31_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-36-31_/.hydra/hydra.yaml b/runs/2025-10-11/21-36-31_/.hydra/hydra.yaml new file mode 100644 index 0000000..c67780b --- /dev/null +++ b/runs/2025-10-11/21-36-31_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-36-31_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-36-31_/.hydra/overrides.yaml b/runs/2025-10-11/21-36-31_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-36-31_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-36-31_/main.log b/runs/2025-10-11/21-36-31_/main.log new file mode 100644 index 0000000..301ebde --- /dev/null +++ b/runs/2025-10-11/21-36-31_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:36:33,710][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-40-18_/.hydra/config.yaml b/runs/2025-10-11/21-40-18_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-40-18_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-40-18_/.hydra/hydra.yaml b/runs/2025-10-11/21-40-18_/.hydra/hydra.yaml new file mode 100644 index 0000000..320d1af --- /dev/null +++ b/runs/2025-10-11/21-40-18_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-40-18_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-40-18_/.hydra/overrides.yaml b/runs/2025-10-11/21-40-18_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-40-18_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-40-18_/main.log b/runs/2025-10-11/21-40-18_/main.log new file mode 100644 index 0000000..2aafaa0 --- /dev/null +++ b/runs/2025-10-11/21-40-18_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:40:20,348][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-45-28_/.hydra/config.yaml b/runs/2025-10-11/21-45-28_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-45-28_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-45-28_/.hydra/hydra.yaml b/runs/2025-10-11/21-45-28_/.hydra/hydra.yaml new file mode 100644 index 0000000..39ef0b1 --- /dev/null +++ b/runs/2025-10-11/21-45-28_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-45-28_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-45-28_/.hydra/overrides.yaml b/runs/2025-10-11/21-45-28_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-45-28_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-45-28_/main.log b/runs/2025-10-11/21-45-28_/main.log new file mode 100644 index 0000000..b1ae81b --- /dev/null +++ b/runs/2025-10-11/21-45-28_/main.log @@ -0,0 +1 @@ +[2025-10-11 21:45:30,575][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/21-50-07_/.hydra/config.yaml b/runs/2025-10-11/21-50-07_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-50-07_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-50-07_/.hydra/hydra.yaml b/runs/2025-10-11/21-50-07_/.hydra/hydra.yaml new file mode 100644 index 0000000..8c171f9 --- /dev/null +++ b/runs/2025-10-11/21-50-07_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-50-07_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-50-07_/.hydra/overrides.yaml b/runs/2025-10-11/21-50-07_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-50-07_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-50-07_/main.log b/runs/2025-10-11/21-50-07_/main.log new file mode 100644 index 0000000..90e88a8 --- /dev/null +++ b/runs/2025-10-11/21-50-07_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 21:50:08,725][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 21:50:09,034][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.tensor(reward) + diff --git a/runs/2025-10-11/21-53-58_/.hydra/config.yaml b/runs/2025-10-11/21-53-58_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-53-58_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-53-58_/.hydra/hydra.yaml b/runs/2025-10-11/21-53-58_/.hydra/hydra.yaml new file mode 100644 index 0000000..9039273 --- /dev/null +++ b/runs/2025-10-11/21-53-58_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-53-58_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-53-58_/.hydra/overrides.yaml b/runs/2025-10-11/21-53-58_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-53-58_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-53-58_/main.log b/runs/2025-10-11/21-53-58_/main.log new file mode 100644 index 0000000..f840c37 --- /dev/null +++ b/runs/2025-10-11/21-53-58_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 21:53:59,807][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 21:54:00,097][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.tensor(reward) + +[2025-10-11 21:54:00,097][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:67: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return self.q_net(torch.tensor(state).to(self.device)) + diff --git a/runs/2025-10-11/21-55-22_/.hydra/config.yaml b/runs/2025-10-11/21-55-22_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-55-22_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-55-22_/.hydra/hydra.yaml b/runs/2025-10-11/21-55-22_/.hydra/hydra.yaml new file mode 100644 index 0000000..edf1209 --- /dev/null +++ b/runs/2025-10-11/21-55-22_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-55-22_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-55-22_/.hydra/overrides.yaml b/runs/2025-10-11/21-55-22_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-55-22_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-55-22_/main.log b/runs/2025-10-11/21-55-22_/main.log new file mode 100644 index 0000000..9d4adac --- /dev/null +++ b/runs/2025-10-11/21-55-22_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 21:55:24,655][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 21:55:24,939][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.tensor(reward) + diff --git a/runs/2025-10-11/21-56-00_/.hydra/config.yaml b/runs/2025-10-11/21-56-00_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-56-00_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-56-00_/.hydra/hydra.yaml b/runs/2025-10-11/21-56-00_/.hydra/hydra.yaml new file mode 100644 index 0000000..f8cfec3 --- /dev/null +++ b/runs/2025-10-11/21-56-00_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-56-00_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-56-00_/.hydra/overrides.yaml b/runs/2025-10-11/21-56-00_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-56-00_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-56-00_/main.log b/runs/2025-10-11/21-56-00_/main.log new file mode 100644 index 0000000..ca56fa0 --- /dev/null +++ b/runs/2025-10-11/21-56-00_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 21:56:02,411][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 21:56:02,697][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.tensor(reward) + diff --git a/runs/2025-10-11/21-56-26_/.hydra/config.yaml b/runs/2025-10-11/21-56-26_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/21-56-26_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/21-56-26_/.hydra/hydra.yaml b/runs/2025-10-11/21-56-26_/.hydra/hydra.yaml new file mode 100644 index 0000000..09d09e6 --- /dev/null +++ b/runs/2025-10-11/21-56-26_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\21-56-26_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/21-56-26_/.hydra/overrides.yaml b/runs/2025-10-11/21-56-26_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/21-56-26_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/21-56-26_/main.log b/runs/2025-10-11/21-56-26_/main.log new file mode 100644 index 0000000..782122a --- /dev/null +++ b/runs/2025-10-11/21-56-26_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 21:56:28,491][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 21:56:28,783][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.tensor(reward) + diff --git a/runs/2025-10-11/22-00-13_/.hydra/config.yaml b/runs/2025-10-11/22-00-13_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-00-13_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-00-13_/.hydra/hydra.yaml b/runs/2025-10-11/22-00-13_/.hydra/hydra.yaml new file mode 100644 index 0000000..cd874c4 --- /dev/null +++ b/runs/2025-10-11/22-00-13_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-00-13_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-00-13_/.hydra/overrides.yaml b/runs/2025-10-11/22-00-13_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-00-13_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-00-13_/main.log b/runs/2025-10-11/22-00-13_/main.log new file mode 100644 index 0000000..268736e --- /dev/null +++ b/runs/2025-10-11/22-00-13_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:00:15,814][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:00:16,078][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + np.where(done, torch.tensor(reward), torch.tensor(reward) + self.gamma * torch.max(self.target_net(torch.tensor(next_state).to(self.device)))) + diff --git a/runs/2025-10-11/22-00-36_/.hydra/config.yaml b/runs/2025-10-11/22-00-36_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-00-36_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-00-36_/.hydra/hydra.yaml b/runs/2025-10-11/22-00-36_/.hydra/hydra.yaml new file mode 100644 index 0000000..64532e3 --- /dev/null +++ b/runs/2025-10-11/22-00-36_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-00-36_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-00-36_/.hydra/overrides.yaml b/runs/2025-10-11/22-00-36_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-00-36_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-00-36_/main.log b/runs/2025-10-11/22-00-36_/main.log new file mode 100644 index 0000000..a61aac0 --- /dev/null +++ b/runs/2025-10-11/22-00-36_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:00:37,780][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:00:38,052][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward) + self.gamma * torch.max(self.target_net(torch.tensor(next_state).to(self.device)))) + diff --git a/runs/2025-10-11/22-02-56_/.hydra/config.yaml b/runs/2025-10-11/22-02-56_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-02-56_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-02-56_/.hydra/hydra.yaml b/runs/2025-10-11/22-02-56_/.hydra/hydra.yaml new file mode 100644 index 0000000..cc98148 --- /dev/null +++ b/runs/2025-10-11/22-02-56_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-02-56_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-02-56_/.hydra/overrides.yaml b/runs/2025-10-11/22-02-56_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-02-56_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-02-56_/main.log b/runs/2025-10-11/22-02-56_/main.log new file mode 100644 index 0000000..8a98451 --- /dev/null +++ b/runs/2025-10-11/22-02-56_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:02:58,218][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:02:58,512][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:56: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device)))* self.gamma) + diff --git a/runs/2025-10-11/22-03-56_/.hydra/config.yaml b/runs/2025-10-11/22-03-56_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-03-56_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-03-56_/.hydra/hydra.yaml b/runs/2025-10-11/22-03-56_/.hydra/hydra.yaml new file mode 100644 index 0000000..ed15827 --- /dev/null +++ b/runs/2025-10-11/22-03-56_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-03-56_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-03-56_/.hydra/overrides.yaml b/runs/2025-10-11/22-03-56_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-03-56_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-03-56_/main.log b/runs/2025-10-11/22-03-56_/main.log new file mode 100644 index 0000000..8805bba --- /dev/null +++ b/runs/2025-10-11/22-03-56_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:03:57,807][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:03:58,085][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr=np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device))) * self.gamma) + diff --git a/runs/2025-10-11/22-04-28_/.hydra/config.yaml b/runs/2025-10-11/22-04-28_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-04-28_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-04-28_/.hydra/hydra.yaml b/runs/2025-10-11/22-04-28_/.hydra/hydra.yaml new file mode 100644 index 0000000..3bca7b2 --- /dev/null +++ b/runs/2025-10-11/22-04-28_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-04-28_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-04-28_/.hydra/overrides.yaml b/runs/2025-10-11/22-04-28_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-04-28_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-04-28_/main.log b/runs/2025-10-11/22-04-28_/main.log new file mode 100644 index 0000000..b03fab0 --- /dev/null +++ b/runs/2025-10-11/22-04-28_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:04:30,233][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:04:30,502][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr=np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device))) * torch.tensor(self.gamma).to(self.device)) + diff --git a/runs/2025-10-11/22-05-35_/.hydra/config.yaml b/runs/2025-10-11/22-05-35_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-05-35_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-05-35_/.hydra/hydra.yaml b/runs/2025-10-11/22-05-35_/.hydra/hydra.yaml new file mode 100644 index 0000000..d78b3d6 --- /dev/null +++ b/runs/2025-10-11/22-05-35_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-05-35_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-05-35_/.hydra/overrides.yaml b/runs/2025-10-11/22-05-35_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-05-35_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-05-35_/main.log b/runs/2025-10-11/22-05-35_/main.log new file mode 100644 index 0000000..84c97dd --- /dev/null +++ b/runs/2025-10-11/22-05-35_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:05:36,943][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:05:37,215][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr=np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device))).to(self.device) * torch.tensor(self.gamma).to(self.device)) + diff --git a/runs/2025-10-11/22-06-44_/.hydra/config.yaml b/runs/2025-10-11/22-06-44_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-06-44_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-06-44_/.hydra/hydra.yaml b/runs/2025-10-11/22-06-44_/.hydra/hydra.yaml new file mode 100644 index 0000000..8101bf5 --- /dev/null +++ b/runs/2025-10-11/22-06-44_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-06-44_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-06-44_/.hydra/overrides.yaml b/runs/2025-10-11/22-06-44_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-06-44_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-06-44_/main.log b/runs/2025-10-11/22-06-44_/main.log new file mode 100644 index 0000000..3b1819c --- /dev/null +++ b/runs/2025-10-11/22-06-44_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:06:46,075][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:06:46,370][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr=np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device)).cpu(), dim=1).to * torch.tensor(self.gamma).to(self.device)) + diff --git a/runs/2025-10-11/22-07-02_/.hydra/config.yaml b/runs/2025-10-11/22-07-02_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-07-02_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-07-02_/.hydra/hydra.yaml b/runs/2025-10-11/22-07-02_/.hydra/hydra.yaml new file mode 100644 index 0000000..620925f --- /dev/null +++ b/runs/2025-10-11/22-07-02_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-07-02_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-07-02_/.hydra/overrides.yaml b/runs/2025-10-11/22-07-02_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-07-02_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-07-02_/main.log b/runs/2025-10-11/22-07-02_/main.log new file mode 100644 index 0000000..48e41f6 --- /dev/null +++ b/runs/2025-10-11/22-07-02_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:07:04,219][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:07:04,537][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr=np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device)).cpu(), dim=1).to(self.device) * torch.tensor(self.gamma).to(self.device)) + diff --git a/runs/2025-10-11/22-07-48_/.hydra/config.yaml b/runs/2025-10-11/22-07-48_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-07-48_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-07-48_/.hydra/hydra.yaml b/runs/2025-10-11/22-07-48_/.hydra/hydra.yaml new file mode 100644 index 0000000..b903472 --- /dev/null +++ b/runs/2025-10-11/22-07-48_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-07-48_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-07-48_/.hydra/overrides.yaml b/runs/2025-10-11/22-07-48_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-07-48_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-07-48_/main.log b/runs/2025-10-11/22-07-48_/main.log new file mode 100644 index 0000000..cdf4997 --- /dev/null +++ b/runs/2025-10-11/22-07-48_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:07:51,021][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:07:51,346][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr=np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + torch.max(self.target_net(torch.tensor(next_state).to(self.device)).cpu(), dim=1) * torch.tensor(self.gamma).to(self.device)) + diff --git a/runs/2025-10-11/22-08-45_/.hydra/config.yaml b/runs/2025-10-11/22-08-45_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-08-45_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-08-45_/.hydra/hydra.yaml b/runs/2025-10-11/22-08-45_/.hydra/hydra.yaml new file mode 100644 index 0000000..b934d53 --- /dev/null +++ b/runs/2025-10-11/22-08-45_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-08-45_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-08-45_/.hydra/overrides.yaml b/runs/2025-10-11/22-08-45_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-08-45_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-08-45_/main.log b/runs/2025-10-11/22-08-45_/main.log new file mode 100644 index 0000000..576d652 --- /dev/null +++ b/runs/2025-10-11/22-08-45_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 22:08:47,695][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:08:48,005][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + max_tensor = torch.max(self.target_net(torch.tensor(next_state).to(self.device)).cpu(), dim=1) + +[2025-10-11 22:08:48,032][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:56: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr = np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + max_tensor * torch.tensor(self.gamma).to(self.device)) + diff --git a/runs/2025-10-11/22-09-31_/.hydra/config.yaml b/runs/2025-10-11/22-09-31_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-09-31_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-09-31_/.hydra/hydra.yaml b/runs/2025-10-11/22-09-31_/.hydra/hydra.yaml new file mode 100644 index 0000000..161ee0b --- /dev/null +++ b/runs/2025-10-11/22-09-31_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-09-31_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-09-31_/.hydra/overrides.yaml b/runs/2025-10-11/22-09-31_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-09-31_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-09-31_/main.log b/runs/2025-10-11/22-09-31_/main.log new file mode 100644 index 0000000..182d394 --- /dev/null +++ b/runs/2025-10-11/22-09-31_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 22:09:33,607][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:09:33,902][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:55: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + max_tensor = torch.max(self.target_net(torch.tensor(next_state).to(self.device)).cpu(), dim=1) + +[2025-10-11 22:09:33,932][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:57: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + tensor_arr = np.where(done, torch.tensor(reward).to(self.device), torch.tensor(reward).to(self.device) + max_tensor * gamma_tensor) + diff --git a/runs/2025-10-11/22-11-26_/.hydra/config.yaml b/runs/2025-10-11/22-11-26_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-11-26_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-11-26_/.hydra/hydra.yaml b/runs/2025-10-11/22-11-26_/.hydra/hydra.yaml new file mode 100644 index 0000000..dc1d363 --- /dev/null +++ b/runs/2025-10-11/22-11-26_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-11-26_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-11-26_/.hydra/overrides.yaml b/runs/2025-10-11/22-11-26_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-11-26_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-11-26_/main.log b/runs/2025-10-11/22-11-26_/main.log new file mode 100644 index 0000000..35209ad --- /dev/null +++ b/runs/2025-10-11/22-11-26_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 22:11:28,537][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:11:28,830][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:56: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + reward_tensor = torch.tensor(reward).to(self.device) + +[2025-10-11 22:11:28,830][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:57: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + max_tensor = torch.max(self.target_net(torch.tensor(next_state).to(self.device)).cpu(), dim=1) + diff --git a/runs/2025-10-11/22-13-11_/.hydra/config.yaml b/runs/2025-10-11/22-13-11_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-13-11_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-13-11_/.hydra/hydra.yaml b/runs/2025-10-11/22-13-11_/.hydra/hydra.yaml new file mode 100644 index 0000000..dc3b98b --- /dev/null +++ b/runs/2025-10-11/22-13-11_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-13-11_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-13-11_/.hydra/overrides.yaml b/runs/2025-10-11/22-13-11_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-13-11_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-13-11_/main.log b/runs/2025-10-11/22-13-11_/main.log new file mode 100644 index 0000000..560bc35 --- /dev/null +++ b/runs/2025-10-11/22-13-11_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:13:13,125][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-13-57_/.hydra/config.yaml b/runs/2025-10-11/22-13-57_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-13-57_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-13-57_/.hydra/hydra.yaml b/runs/2025-10-11/22-13-57_/.hydra/hydra.yaml new file mode 100644 index 0000000..30df55a --- /dev/null +++ b/runs/2025-10-11/22-13-57_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-13-57_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-13-57_/.hydra/overrides.yaml b/runs/2025-10-11/22-13-57_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-13-57_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-13-57_/main.log b/runs/2025-10-11/22-13-57_/main.log new file mode 100644 index 0000000..1141cab --- /dev/null +++ b/runs/2025-10-11/22-13-57_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:13:59,498][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-14-06_/.hydra/config.yaml b/runs/2025-10-11/22-14-06_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-14-06_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-14-06_/.hydra/hydra.yaml b/runs/2025-10-11/22-14-06_/.hydra/hydra.yaml new file mode 100644 index 0000000..ebc0043 --- /dev/null +++ b/runs/2025-10-11/22-14-06_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-14-06_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-14-06_/.hydra/overrides.yaml b/runs/2025-10-11/22-14-06_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-14-06_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-14-06_/main.log b/runs/2025-10-11/22-14-06_/main.log new file mode 100644 index 0000000..dfd7c4c --- /dev/null +++ b/runs/2025-10-11/22-14-06_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:14:08,522][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-18-07_/.hydra/config.yaml b/runs/2025-10-11/22-18-07_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-18-07_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-18-07_/.hydra/hydra.yaml b/runs/2025-10-11/22-18-07_/.hydra/hydra.yaml new file mode 100644 index 0000000..5d64783 --- /dev/null +++ b/runs/2025-10-11/22-18-07_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-18-07_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-18-07_/.hydra/overrides.yaml b/runs/2025-10-11/22-18-07_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-18-07_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-18-07_/main.log b/runs/2025-10-11/22-18-07_/main.log new file mode 100644 index 0000000..69923cc --- /dev/null +++ b/runs/2025-10-11/22-18-07_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:18:09,252][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-18-54_/.hydra/config.yaml b/runs/2025-10-11/22-18-54_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-18-54_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-18-54_/.hydra/hydra.yaml b/runs/2025-10-11/22-18-54_/.hydra/hydra.yaml new file mode 100644 index 0000000..9af89a4 --- /dev/null +++ b/runs/2025-10-11/22-18-54_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-18-54_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-18-54_/.hydra/overrides.yaml b/runs/2025-10-11/22-18-54_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-18-54_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-18-54_/main.log b/runs/2025-10-11/22-18-54_/main.log new file mode 100644 index 0000000..07e85c6 --- /dev/null +++ b/runs/2025-10-11/22-18-54_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:18:56,252][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-21-12_/.hydra/config.yaml b/runs/2025-10-11/22-21-12_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-21-12_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-21-12_/.hydra/hydra.yaml b/runs/2025-10-11/22-21-12_/.hydra/hydra.yaml new file mode 100644 index 0000000..077b35a --- /dev/null +++ b/runs/2025-10-11/22-21-12_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-21-12_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-21-12_/.hydra/overrides.yaml b/runs/2025-10-11/22-21-12_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-21-12_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-21-12_/main.log b/runs/2025-10-11/22-21-12_/main.log new file mode 100644 index 0000000..166713c --- /dev/null +++ b/runs/2025-10-11/22-21-12_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:21:13,907][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-22-03_/.hydra/config.yaml b/runs/2025-10-11/22-22-03_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-22-03_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-22-03_/.hydra/hydra.yaml b/runs/2025-10-11/22-22-03_/.hydra/hydra.yaml new file mode 100644 index 0000000..1ca0d21 --- /dev/null +++ b/runs/2025-10-11/22-22-03_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-22-03_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-22-03_/.hydra/overrides.yaml b/runs/2025-10-11/22-22-03_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-22-03_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-22-03_/main.log b/runs/2025-10-11/22-22-03_/main.log new file mode 100644 index 0000000..8b52de3 --- /dev/null +++ b/runs/2025-10-11/22-22-03_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:22:04,843][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-23-22_/.hydra/config.yaml b/runs/2025-10-11/22-23-22_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-23-22_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-23-22_/.hydra/hydra.yaml b/runs/2025-10-11/22-23-22_/.hydra/hydra.yaml new file mode 100644 index 0000000..7d1fb1a --- /dev/null +++ b/runs/2025-10-11/22-23-22_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-23-22_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-23-22_/.hydra/overrides.yaml b/runs/2025-10-11/22-23-22_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-23-22_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-23-22_/main.log b/runs/2025-10-11/22-23-22_/main.log new file mode 100644 index 0000000..0660fea --- /dev/null +++ b/runs/2025-10-11/22-23-22_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:23:24,352][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-24-15_/.hydra/config.yaml b/runs/2025-10-11/22-24-15_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-24-15_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-24-15_/.hydra/hydra.yaml b/runs/2025-10-11/22-24-15_/.hydra/hydra.yaml new file mode 100644 index 0000000..82559f5 --- /dev/null +++ b/runs/2025-10-11/22-24-15_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-24-15_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-24-15_/.hydra/overrides.yaml b/runs/2025-10-11/22-24-15_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-24-15_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-24-15_/main.log b/runs/2025-10-11/22-24-15_/main.log new file mode 100644 index 0000000..79070ec --- /dev/null +++ b/runs/2025-10-11/22-24-15_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:24:17,523][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-24-56_/.hydra/config.yaml b/runs/2025-10-11/22-24-56_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-24-56_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-24-56_/.hydra/hydra.yaml b/runs/2025-10-11/22-24-56_/.hydra/hydra.yaml new file mode 100644 index 0000000..2757d80 --- /dev/null +++ b/runs/2025-10-11/22-24-56_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-24-56_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-24-56_/.hydra/overrides.yaml b/runs/2025-10-11/22-24-56_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-24-56_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-24-56_/main.log b/runs/2025-10-11/22-24-56_/main.log new file mode 100644 index 0000000..0ca40b8 --- /dev/null +++ b/runs/2025-10-11/22-24-56_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:24:58,084][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-26-01_/.hydra/config.yaml b/runs/2025-10-11/22-26-01_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-26-01_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-26-01_/.hydra/hydra.yaml b/runs/2025-10-11/22-26-01_/.hydra/hydra.yaml new file mode 100644 index 0000000..30738bf --- /dev/null +++ b/runs/2025-10-11/22-26-01_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-26-01_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-26-01_/.hydra/overrides.yaml b/runs/2025-10-11/22-26-01_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-26-01_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-26-01_/main.log b/runs/2025-10-11/22-26-01_/main.log new file mode 100644 index 0000000..15d45be --- /dev/null +++ b/runs/2025-10-11/22-26-01_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:26:03,056][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-27-28_/.hydra/config.yaml b/runs/2025-10-11/22-27-28_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-27-28_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-27-28_/.hydra/hydra.yaml b/runs/2025-10-11/22-27-28_/.hydra/hydra.yaml new file mode 100644 index 0000000..fd1852d --- /dev/null +++ b/runs/2025-10-11/22-27-28_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-27-28_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-27-28_/.hydra/overrides.yaml b/runs/2025-10-11/22-27-28_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-27-28_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-27-28_/main.log b/runs/2025-10-11/22-27-28_/main.log new file mode 100644 index 0000000..58bf9e0 --- /dev/null +++ b/runs/2025-10-11/22-27-28_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:27:30,449][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-27-48_/.hydra/config.yaml b/runs/2025-10-11/22-27-48_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-27-48_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-27-48_/.hydra/hydra.yaml b/runs/2025-10-11/22-27-48_/.hydra/hydra.yaml new file mode 100644 index 0000000..1adc2cd --- /dev/null +++ b/runs/2025-10-11/22-27-48_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-27-48_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-27-48_/.hydra/overrides.yaml b/runs/2025-10-11/22-27-48_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-27-48_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-27-48_/main.log b/runs/2025-10-11/22-27-48_/main.log new file mode 100644 index 0000000..547dc75 --- /dev/null +++ b/runs/2025-10-11/22-27-48_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:27:50,253][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-29-04_/.hydra/config.yaml b/runs/2025-10-11/22-29-04_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-29-04_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-29-04_/.hydra/hydra.yaml b/runs/2025-10-11/22-29-04_/.hydra/hydra.yaml new file mode 100644 index 0000000..1fbe125 --- /dev/null +++ b/runs/2025-10-11/22-29-04_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-29-04_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-29-04_/.hydra/overrides.yaml b/runs/2025-10-11/22-29-04_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-29-04_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-29-04_/main.log b/runs/2025-10-11/22-29-04_/main.log new file mode 100644 index 0000000..168f198 --- /dev/null +++ b/runs/2025-10-11/22-29-04_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:29:06,221][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:29:06,689][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:72: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.index_select(q, 1, torch.tensor(action).to(self.device)) + diff --git a/runs/2025-10-11/22-29-38_/.hydra/config.yaml b/runs/2025-10-11/22-29-38_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-29-38_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-29-38_/.hydra/hydra.yaml b/runs/2025-10-11/22-29-38_/.hydra/hydra.yaml new file mode 100644 index 0000000..ac6d888 --- /dev/null +++ b/runs/2025-10-11/22-29-38_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-29-38_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-29-38_/.hydra/overrides.yaml b/runs/2025-10-11/22-29-38_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-29-38_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-29-38_/main.log b/runs/2025-10-11/22-29-38_/main.log new file mode 100644 index 0000000..c6d9fba --- /dev/null +++ b/runs/2025-10-11/22-29-38_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:29:40,178][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:29:40,628][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:72: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.index_select(q, 1, torch.tensor(action)) + diff --git a/runs/2025-10-11/22-29-54_/.hydra/config.yaml b/runs/2025-10-11/22-29-54_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-29-54_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-29-54_/.hydra/hydra.yaml b/runs/2025-10-11/22-29-54_/.hydra/hydra.yaml new file mode 100644 index 0000000..1a18124 --- /dev/null +++ b/runs/2025-10-11/22-29-54_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-29-54_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-29-54_/.hydra/overrides.yaml b/runs/2025-10-11/22-29-54_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-29-54_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-29-54_/main.log b/runs/2025-10-11/22-29-54_/main.log new file mode 100644 index 0000000..34bf1a5 --- /dev/null +++ b/runs/2025-10-11/22-29-54_/main.log @@ -0,0 +1,4 @@ +[2025-10-11 22:29:56,117][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:29:56,551][py.warnings][WARNING] - d:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\agent.py:72: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + return torch.index_select(q, 1, torch.tensor(action).to(self.device)) + diff --git a/runs/2025-10-11/22-30-28_/.hydra/config.yaml b/runs/2025-10-11/22-30-28_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-30-28_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-30-28_/.hydra/hydra.yaml b/runs/2025-10-11/22-30-28_/.hydra/hydra.yaml new file mode 100644 index 0000000..633b16b --- /dev/null +++ b/runs/2025-10-11/22-30-28_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-30-28_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-30-28_/.hydra/overrides.yaml b/runs/2025-10-11/22-30-28_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-30-28_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-30-28_/main.log b/runs/2025-10-11/22-30-28_/main.log new file mode 100644 index 0000000..ca86223 --- /dev/null +++ b/runs/2025-10-11/22-30-28_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:30:30,043][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-30-49_/.hydra/config.yaml b/runs/2025-10-11/22-30-49_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-30-49_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-30-49_/.hydra/hydra.yaml b/runs/2025-10-11/22-30-49_/.hydra/hydra.yaml new file mode 100644 index 0000000..e647cfe --- /dev/null +++ b/runs/2025-10-11/22-30-49_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-30-49_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-30-49_/.hydra/overrides.yaml b/runs/2025-10-11/22-30-49_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-30-49_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-30-49_/main.log b/runs/2025-10-11/22-30-49_/main.log new file mode 100644 index 0000000..672fe3a --- /dev/null +++ b/runs/2025-10-11/22-30-49_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:30:51,155][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-31-21_/.hydra/config.yaml b/runs/2025-10-11/22-31-21_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-31-21_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-31-21_/.hydra/hydra.yaml b/runs/2025-10-11/22-31-21_/.hydra/hydra.yaml new file mode 100644 index 0000000..614ca17 --- /dev/null +++ b/runs/2025-10-11/22-31-21_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-31-21_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-31-21_/.hydra/overrides.yaml b/runs/2025-10-11/22-31-21_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-31-21_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-31-21_/main.log b/runs/2025-10-11/22-31-21_/main.log new file mode 100644 index 0000000..5955f5c --- /dev/null +++ b/runs/2025-10-11/22-31-21_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:31:22,913][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-32-18_/.hydra/config.yaml b/runs/2025-10-11/22-32-18_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-32-18_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-32-18_/.hydra/hydra.yaml b/runs/2025-10-11/22-32-18_/.hydra/hydra.yaml new file mode 100644 index 0000000..7b67c53 --- /dev/null +++ b/runs/2025-10-11/22-32-18_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-32-18_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-32-18_/.hydra/overrides.yaml b/runs/2025-10-11/22-32-18_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-32-18_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-32-18_/main.log b/runs/2025-10-11/22-32-18_/main.log new file mode 100644 index 0000000..cf1bcb2 --- /dev/null +++ b/runs/2025-10-11/22-32-18_/main.log @@ -0,0 +1,2 @@ +[2025-10-11 22:32:20,571][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:32:37,605][core][INFO] - Step: 2000, Eval mean: 9.2, Eval std: 0.6 diff --git a/runs/2025-10-11/22-32-18_/models/best_model.pt b/runs/2025-10-11/22-32-18_/models/best_model.pt new file mode 100644 index 0000000..1c67c54 Binary files /dev/null and b/runs/2025-10-11/22-32-18_/models/best_model.pt differ diff --git a/runs/2025-10-11/22-32-18_/results.png b/runs/2025-10-11/22-32-18_/results.png new file mode 100644 index 0000000..7d095a5 Binary files /dev/null and b/runs/2025-10-11/22-32-18_/results.png differ diff --git a/runs/2025-10-11/22-32-57_/.hydra/config.yaml b/runs/2025-10-11/22-32-57_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-32-57_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-32-57_/.hydra/hydra.yaml b/runs/2025-10-11/22-32-57_/.hydra/hydra.yaml new file mode 100644 index 0000000..d3bd414 --- /dev/null +++ b/runs/2025-10-11/22-32-57_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-32-57_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-32-57_/.hydra/overrides.yaml b/runs/2025-10-11/22-32-57_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-32-57_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-32-57_/main.log b/runs/2025-10-11/22-32-57_/main.log new file mode 100644 index 0000000..db92fe8 --- /dev/null +++ b/runs/2025-10-11/22-32-57_/main.log @@ -0,0 +1,14 @@ +[2025-10-11 22:32:58,892][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:33:08,427][core][INFO] - Step: 2000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:33:19,195][core][INFO] - Step: 4000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:33:29,736][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:33:40,937][core][INFO] - Step: 8000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:33:52,238][core][INFO] - Step: 10000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:34:03,847][core][INFO] - Step: 12000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:34:15,649][core][INFO] - Step: 14000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:34:28,471][core][INFO] - Step: 16000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:34:41,494][core][INFO] - Step: 18000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:34:53,973][core][INFO] - Step: 20000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:35:06,951][core][INFO] - Step: 22000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:35:19,579][core][INFO] - Step: 24000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:35:33,048][core][INFO] - Step: 26000, Eval mean: 9.4, Eval std: 0.66332495807108 diff --git a/runs/2025-10-11/22-32-57_/models/best_model.pt b/runs/2025-10-11/22-32-57_/models/best_model.pt new file mode 100644 index 0000000..a3f2adf Binary files /dev/null and b/runs/2025-10-11/22-32-57_/models/best_model.pt differ diff --git a/runs/2025-10-11/22-32-57_/results.png b/runs/2025-10-11/22-32-57_/results.png new file mode 100644 index 0000000..97296cf Binary files /dev/null and b/runs/2025-10-11/22-32-57_/results.png differ diff --git a/runs/2025-10-11/22-35-41_/.hydra/config.yaml b/runs/2025-10-11/22-35-41_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-35-41_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-35-41_/.hydra/hydra.yaml b/runs/2025-10-11/22-35-41_/.hydra/hydra.yaml new file mode 100644 index 0000000..a814dd8 --- /dev/null +++ b/runs/2025-10-11/22-35-41_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-35-41_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-35-41_/.hydra/overrides.yaml b/runs/2025-10-11/22-35-41_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-35-41_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-35-41_/main.log b/runs/2025-10-11/22-35-41_/main.log new file mode 100644 index 0000000..571b2de --- /dev/null +++ b/runs/2025-10-11/22-35-41_/main.log @@ -0,0 +1,12 @@ +[2025-10-11 22:35:43,673][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:35:53,175][core][INFO] - Step: 2000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:36:04,147][core][INFO] - Step: 4000, Eval mean: 25.2, Eval std: 2.1354156504062622 +[2025-10-11 22:36:14,701][core][INFO] - Step: 6000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:36:25,593][core][INFO] - Step: 8000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:36:36,771][core][INFO] - Step: 10000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:36:48,501][core][INFO] - Step: 12000, Eval mean: 13.2, Eval std: 0.9797958971132712 +[2025-10-11 22:37:00,186][core][INFO] - Step: 14000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:37:12,253][core][INFO] - Step: 16000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:37:24,363][core][INFO] - Step: 18000, Eval mean: 12.6, Eval std: 1.4966629547095764 +[2025-10-11 22:37:37,055][core][INFO] - Step: 20000, Eval mean: 12.9, Eval std: 1.0440306508910548 +[2025-10-11 22:37:49,427][core][INFO] - Step: 22000, Eval mean: 9.2, Eval std: 0.6 diff --git a/runs/2025-10-11/22-35-41_/models/best_model.pt b/runs/2025-10-11/22-35-41_/models/best_model.pt new file mode 100644 index 0000000..cffb40d Binary files /dev/null and b/runs/2025-10-11/22-35-41_/models/best_model.pt differ diff --git a/runs/2025-10-11/22-35-41_/results.png b/runs/2025-10-11/22-35-41_/results.png new file mode 100644 index 0000000..e441da2 Binary files /dev/null and b/runs/2025-10-11/22-35-41_/results.png differ diff --git a/runs/2025-10-11/22-39-44_/.hydra/config.yaml b/runs/2025-10-11/22-39-44_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-39-44_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-39-44_/.hydra/hydra.yaml b/runs/2025-10-11/22-39-44_/.hydra/hydra.yaml new file mode 100644 index 0000000..aff9313 --- /dev/null +++ b/runs/2025-10-11/22-39-44_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-39-44_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-39-44_/.hydra/overrides.yaml b/runs/2025-10-11/22-39-44_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-39-44_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-39-44_/main.log b/runs/2025-10-11/22-39-44_/main.log new file mode 100644 index 0000000..3f99551 --- /dev/null +++ b/runs/2025-10-11/22-39-44_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:39:45,863][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-43-29_/.hydra/config.yaml b/runs/2025-10-11/22-43-29_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-43-29_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-43-29_/.hydra/hydra.yaml b/runs/2025-10-11/22-43-29_/.hydra/hydra.yaml new file mode 100644 index 0000000..6b09633 --- /dev/null +++ b/runs/2025-10-11/22-43-29_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-43-29_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-43-29_/.hydra/overrides.yaml b/runs/2025-10-11/22-43-29_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-43-29_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-43-29_/main.log b/runs/2025-10-11/22-43-29_/main.log new file mode 100644 index 0000000..65a01e1 --- /dev/null +++ b/runs/2025-10-11/22-43-29_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:43:31,673][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-44-12_/.hydra/config.yaml b/runs/2025-10-11/22-44-12_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-44-12_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-44-12_/.hydra/hydra.yaml b/runs/2025-10-11/22-44-12_/.hydra/hydra.yaml new file mode 100644 index 0000000..cea2c2a --- /dev/null +++ b/runs/2025-10-11/22-44-12_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-44-12_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-44-12_/.hydra/overrides.yaml b/runs/2025-10-11/22-44-12_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-44-12_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-44-12_/main.log b/runs/2025-10-11/22-44-12_/main.log new file mode 100644 index 0000000..95ae51f --- /dev/null +++ b/runs/2025-10-11/22-44-12_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:44:14,330][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-44-30_/.hydra/config.yaml b/runs/2025-10-11/22-44-30_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-44-30_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-44-30_/.hydra/hydra.yaml b/runs/2025-10-11/22-44-30_/.hydra/hydra.yaml new file mode 100644 index 0000000..bb7eb34 --- /dev/null +++ b/runs/2025-10-11/22-44-30_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-44-30_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-44-30_/.hydra/overrides.yaml b/runs/2025-10-11/22-44-30_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-44-30_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-44-30_/main.log b/runs/2025-10-11/22-44-30_/main.log new file mode 100644 index 0000000..1b79656 --- /dev/null +++ b/runs/2025-10-11/22-44-30_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:44:31,787][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-45-48_/.hydra/config.yaml b/runs/2025-10-11/22-45-48_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-45-48_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-45-48_/.hydra/hydra.yaml b/runs/2025-10-11/22-45-48_/.hydra/hydra.yaml new file mode 100644 index 0000000..a6e87fa --- /dev/null +++ b/runs/2025-10-11/22-45-48_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-45-48_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-45-48_/.hydra/overrides.yaml b/runs/2025-10-11/22-45-48_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-45-48_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-45-48_/main.log b/runs/2025-10-11/22-45-48_/main.log new file mode 100644 index 0000000..7654546 --- /dev/null +++ b/runs/2025-10-11/22-45-48_/main.log @@ -0,0 +1,15 @@ +[2025-10-11 22:45:50,761][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:46:00,290][core][INFO] - Step: 2000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:46:10,843][core][INFO] - Step: 4000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:46:21,821][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:46:33,067][core][INFO] - Step: 8000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:46:44,081][core][INFO] - Step: 10000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:46:55,719][core][INFO] - Step: 12000, Eval mean: 12.1, Eval std: 2.4269322199023193 +[2025-10-11 22:47:07,463][core][INFO] - Step: 14000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:47:19,411][core][INFO] - Step: 16000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:47:31,631][core][INFO] - Step: 18000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:47:44,763][core][INFO] - Step: 20000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:47:57,803][core][INFO] - Step: 22000, Eval mean: 10.5, Eval std: 0.9219544457292888 +[2025-10-11 22:48:10,465][core][INFO] - Step: 24000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:48:22,940][core][INFO] - Step: 26000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:48:35,661][core][INFO] - Step: 28000, Eval mean: 9.4, Eval std: 0.66332495807108 diff --git a/runs/2025-10-11/22-45-48_/models/best_model.pt b/runs/2025-10-11/22-45-48_/models/best_model.pt new file mode 100644 index 0000000..5566435 Binary files /dev/null and b/runs/2025-10-11/22-45-48_/models/best_model.pt differ diff --git a/runs/2025-10-11/22-45-48_/results.png b/runs/2025-10-11/22-45-48_/results.png new file mode 100644 index 0000000..de0b708 Binary files /dev/null and b/runs/2025-10-11/22-45-48_/results.png differ diff --git a/runs/2025-10-11/22-53-16_/.hydra/config.yaml b/runs/2025-10-11/22-53-16_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-53-16_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-53-16_/.hydra/hydra.yaml b/runs/2025-10-11/22-53-16_/.hydra/hydra.yaml new file mode 100644 index 0000000..7129228 --- /dev/null +++ b/runs/2025-10-11/22-53-16_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-53-16_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-53-16_/.hydra/overrides.yaml b/runs/2025-10-11/22-53-16_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-53-16_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-53-16_/main.log b/runs/2025-10-11/22-53-16_/main.log new file mode 100644 index 0000000..025bb41 --- /dev/null +++ b/runs/2025-10-11/22-53-16_/main.log @@ -0,0 +1 @@ +[2025-10-11 22:53:18,171][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/22-54-18_/.hydra/config.yaml b/runs/2025-10-11/22-54-18_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/22-54-18_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/22-54-18_/.hydra/hydra.yaml b/runs/2025-10-11/22-54-18_/.hydra/hydra.yaml new file mode 100644 index 0000000..a3dc78a --- /dev/null +++ b/runs/2025-10-11/22-54-18_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\22-54-18_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/22-54-18_/.hydra/overrides.yaml b/runs/2025-10-11/22-54-18_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/22-54-18_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/22-54-18_/main.log b/runs/2025-10-11/22-54-18_/main.log new file mode 100644 index 0000000..be1d76a --- /dev/null +++ b/runs/2025-10-11/22-54-18_/main.log @@ -0,0 +1,7 @@ +[2025-10-11 22:54:20,323][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 22:54:29,346][core][INFO] - Step: 2000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:54:39,848][core][INFO] - Step: 4000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:54:50,222][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:55:00,611][core][INFO] - Step: 8000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 22:55:11,300][core][INFO] - Step: 10000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 22:55:22,394][core][INFO] - Step: 12000, Eval mean: 9.4, Eval std: 0.66332495807108 diff --git a/runs/2025-10-11/22-54-18_/models/best_model.pt b/runs/2025-10-11/22-54-18_/models/best_model.pt new file mode 100644 index 0000000..28e8ecb Binary files /dev/null and b/runs/2025-10-11/22-54-18_/models/best_model.pt differ diff --git a/runs/2025-10-11/22-54-18_/results.png b/runs/2025-10-11/22-54-18_/results.png new file mode 100644 index 0000000..4a4d8db Binary files /dev/null and b/runs/2025-10-11/22-54-18_/results.png differ diff --git a/runs/2025-10-11/23-00-28_/.hydra/config.yaml b/runs/2025-10-11/23-00-28_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-00-28_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-00-28_/.hydra/hydra.yaml b/runs/2025-10-11/23-00-28_/.hydra/hydra.yaml new file mode 100644 index 0000000..6ee6082 --- /dev/null +++ b/runs/2025-10-11/23-00-28_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\23-00-28_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-00-28_/.hydra/overrides.yaml b/runs/2025-10-11/23-00-28_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-00-28_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-00-28_/main.log b/runs/2025-10-11/23-00-28_/main.log new file mode 100644 index 0000000..38e4841 --- /dev/null +++ b/runs/2025-10-11/23-00-28_/main.log @@ -0,0 +1,14 @@ +[2025-10-11 23:00:29,935][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 23:00:40,428][core][INFO] - Step: 2000, Eval mean: 262.8, Eval std: 52.92409659125038 +[2025-10-11 23:00:51,892][core][INFO] - Step: 4000, Eval mean: 243.4, Eval std: 65.53655468515262 +[2025-10-11 23:01:03,173][core][INFO] - Step: 6000, Eval mean: 181.4, Eval std: 57.339689570139804 +[2025-10-11 23:01:13,851][core][INFO] - Step: 8000, Eval mean: 98.2, Eval std: 2.638181191654584 +[2025-10-11 23:01:25,103][core][INFO] - Step: 10000, Eval mean: 116.9, Eval std: 5.18555686498567 +[2025-10-11 23:01:36,581][core][INFO] - Step: 12000, Eval mean: 108.0, Eval std: 3.4641016151377544 +[2025-10-11 23:01:48,356][core][INFO] - Step: 14000, Eval mean: 149.4, Eval std: 7.578918128598566 +[2025-10-11 23:02:00,898][core][INFO] - Step: 16000, Eval mean: 250.2, Eval std: 12.416118556135006 +[2025-10-11 23:02:14,704][core][INFO] - Step: 18000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:02:28,706][core][INFO] - Step: 20000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:02:42,852][core][INFO] - Step: 22000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:02:56,994][core][INFO] - Step: 24000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:03:11,660][core][INFO] - Step: 26000, Eval mean: 500.0, Eval std: 0.0 diff --git a/runs/2025-10-11/23-00-28_/models/best_model.pt b/runs/2025-10-11/23-00-28_/models/best_model.pt new file mode 100644 index 0000000..447c587 Binary files /dev/null and b/runs/2025-10-11/23-00-28_/models/best_model.pt differ diff --git a/runs/2025-10-11/23-00-28_/results.png b/runs/2025-10-11/23-00-28_/results.png new file mode 100644 index 0000000..b0dcb7a Binary files /dev/null and b/runs/2025-10-11/23-00-28_/results.png differ diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/config.yaml b/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/config.yaml new file mode 100644 index 0000000..a717712 --- /dev/null +++ b/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: true +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/hydra.yaml b/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/hydra.yaml new file mode 100644 index 0000000..b1e7523 --- /dev/null +++ b/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent.use_double=true + job: + name: main + chdir: true + override_dirname: agent.use_double=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\runs\2025-10-11\23-13-55_agent.use_double=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/overrides.yaml b/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/overrides.yaml new file mode 100644 index 0000000..23a2a31 --- /dev/null +++ b/runs/2025-10-11/23-13-55_agent.use_double=true/.hydra/overrides.yaml @@ -0,0 +1 @@ +- agent.use_double=true diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/best_videos.mp4 b/runs/2025-10-11/23-13-55_agent.use_double=true/best_videos.mp4 new file mode 100644 index 0000000..570b400 Binary files /dev/null and b/runs/2025-10-11/23-13-55_agent.use_double=true/best_videos.mp4 differ diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/final_videos.mp4 b/runs/2025-10-11/23-13-55_agent.use_double=true/final_videos.mp4 new file mode 100644 index 0000000..d203370 Binary files /dev/null and b/runs/2025-10-11/23-13-55_agent.use_double=true/final_videos.mp4 differ diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/main.log b/runs/2025-10-11/23-13-55_agent.use_double=true/main.log new file mode 100644 index 0000000..1d7cf4b --- /dev/null +++ b/runs/2025-10-11/23-13-55_agent.use_double=true/main.log @@ -0,0 +1,28 @@ +[2025-10-11 23:13:57,284][__main__][INFO] - Training for 50000 timesteps with DoubleQNetwork and NormalReplayBuffer +[2025-10-11 23:14:07,969][core][INFO] - Step: 2000, Eval mean: 218.8, Eval std: 46.64933011308951 +[2025-10-11 23:14:20,236][core][INFO] - Step: 4000, Eval mean: 289.1, Eval std: 84.42920110956872 +[2025-10-11 23:14:32,817][core][INFO] - Step: 6000, Eval mean: 318.0, Eval std: 121.90570126126177 +[2025-10-11 23:14:45,386][core][INFO] - Step: 8000, Eval mean: 368.1, Eval std: 90.16479357265783 +[2025-10-11 23:14:58,010][core][INFO] - Step: 10000, Eval mean: 336.9, Eval std: 86.67000634590954 +[2025-10-11 23:15:10,690][core][INFO] - Step: 12000, Eval mean: 307.9, Eval std: 74.98993265765745 +[2025-10-11 23:15:23,173][core][INFO] - Step: 14000, Eval mean: 248.0, Eval std: 49.15689168366934 +[2025-10-11 23:15:35,842][core][INFO] - Step: 16000, Eval mean: 205.7, Eval std: 22.60110616761932 +[2025-10-11 23:15:48,478][core][INFO] - Step: 18000, Eval mean: 140.9, Eval std: 7.608547824650904 +[2025-10-11 23:16:00,748][core][INFO] - Step: 20000, Eval mean: 142.5, Eval std: 6.786015030929419 +[2025-10-11 23:16:13,238][core][INFO] - Step: 22000, Eval mean: 143.7, Eval std: 7.22564875980005 +[2025-10-11 23:16:27,402][core][INFO] - Step: 24000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:16:41,807][core][INFO] - Step: 26000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:16:56,302][core][INFO] - Step: 28000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:17:10,156][core][INFO] - Step: 30000, Eval mean: 317.2, Eval std: 46.84613111026353 +[2025-10-11 23:17:24,824][core][INFO] - Step: 32000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:17:39,347][core][INFO] - Step: 34000, Eval mean: 356.2, Eval std: 49.19512170937277 +[2025-10-11 23:17:55,602][core][INFO] - Step: 36000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:18:11,503][core][INFO] - Step: 38000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:18:27,027][core][INFO] - Step: 40000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:18:42,744][core][INFO] - Step: 42000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:18:58,366][core][INFO] - Step: 44000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:19:15,543][core][INFO] - Step: 46000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:19:31,906][core][INFO] - Step: 48000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:19:48,053][core][INFO] - Step: 50000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:20:09,011][core][INFO] - Final Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:20:14,555][__main__][INFO] - Finish training with eval mean: 500.0 diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/models/best_model.pt b/runs/2025-10-11/23-13-55_agent.use_double=true/models/best_model.pt new file mode 100644 index 0000000..e7903a5 Binary files /dev/null and b/runs/2025-10-11/23-13-55_agent.use_double=true/models/best_model.pt differ diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/models/final_model.pt b/runs/2025-10-11/23-13-55_agent.use_double=true/models/final_model.pt new file mode 100644 index 0000000..d7ca185 Binary files /dev/null and b/runs/2025-10-11/23-13-55_agent.use_double=true/models/final_model.pt differ diff --git a/runs/2025-10-11/23-13-55_agent.use_double=true/results.png b/runs/2025-10-11/23-13-55_agent.use_double=true/results.png new file mode 100644 index 0000000..1997a8a Binary files /dev/null and b/runs/2025-10-11/23-13-55_agent.use_double=true/results.png differ diff --git a/runs/2025-10-11/23-25-31_/.hydra/config.yaml b/runs/2025-10-11/23-25-31_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-25-31_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-25-31_/.hydra/hydra.yaml b/runs/2025-10-11/23-25-31_/.hydra/hydra.yaml new file mode 100644 index 0000000..d659e48 --- /dev/null +++ b/runs/2025-10-11/23-25-31_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-25-31_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-25-31_/.hydra/overrides.yaml b/runs/2025-10-11/23-25-31_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-25-31_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-25-31_/main.log b/runs/2025-10-11/23-25-31_/main.log new file mode 100644 index 0000000..9ff4ed1 --- /dev/null +++ b/runs/2025-10-11/23-25-31_/main.log @@ -0,0 +1,2 @@ +[2025-10-11 23:25:33,757][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 23:25:44,936][core][INFO] - Step: 2000, Eval mean: 262.8, Eval std: 52.92409659125038 diff --git a/runs/2025-10-11/23-25-31_/models/best_model.pt b/runs/2025-10-11/23-25-31_/models/best_model.pt new file mode 100644 index 0000000..55b14cb Binary files /dev/null and b/runs/2025-10-11/23-25-31_/models/best_model.pt differ diff --git a/runs/2025-10-11/23-25-31_/results.png b/runs/2025-10-11/23-25-31_/results.png new file mode 100644 index 0000000..b2a8744 Binary files /dev/null and b/runs/2025-10-11/23-25-31_/results.png differ diff --git a/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/config.yaml b/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/config.yaml new file mode 100644 index 0000000..318a17d --- /dev/null +++ b/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: true + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/hydra.yaml b/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/hydra.yaml new file mode 100644 index 0000000..7b7753d --- /dev/null +++ b/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent.use_dueling=true + job: + name: main + chdir: true + override_dirname: agent.use_dueling=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-26-02_agent.use_dueling=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/overrides.yaml b/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/overrides.yaml new file mode 100644 index 0000000..6c30869 --- /dev/null +++ b/runs/2025-10-11/23-26-02_agent.use_dueling=true/.hydra/overrides.yaml @@ -0,0 +1 @@ +- agent.use_dueling=true diff --git a/runs/2025-10-11/23-26-02_agent.use_dueling=true/main.log b/runs/2025-10-11/23-26-02_agent.use_dueling=true/main.log new file mode 100644 index 0000000..4f9d42f --- /dev/null +++ b/runs/2025-10-11/23-26-02_agent.use_dueling=true/main.log @@ -0,0 +1,15 @@ +[2025-10-11 23:26:03,897][__main__][INFO] - Training for 50000 timesteps with DuelingQNetwork and NormalReplayBuffer +[2025-10-11 23:26:17,366][core][INFO] - Step: 2000, Eval mean: 226.0, Eval std: 42.757455490241696 +[2025-10-11 23:26:31,018][core][INFO] - Step: 4000, Eval mean: 106.6, Eval std: 3.49857113690718 +[2025-10-11 23:26:44,765][core][INFO] - Step: 6000, Eval mean: 111.5, Eval std: 3.1064449134018135 +[2025-10-11 23:26:58,783][core][INFO] - Step: 8000, Eval mean: 96.8, Eval std: 2.821347195933177 +[2025-10-11 23:27:13,637][core][INFO] - Step: 10000, Eval mean: 112.4, Eval std: 4.476605857119878 +[2025-10-11 23:27:30,697][core][INFO] - Step: 12000, Eval mean: 257.3, Eval std: 5.692978130996114 +[2025-10-11 23:27:49,873][core][INFO] - Step: 14000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:28:06,234][core][INFO] - Step: 16000, Eval mean: 113.8, Eval std: 3.515679166249389 +[2025-10-11 23:28:22,582][core][INFO] - Step: 18000, Eval mean: 182.2, Eval std: 8.022468448052631 +[2025-10-11 23:28:41,446][core][INFO] - Step: 20000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:29:00,468][core][INFO] - Step: 22000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:29:19,606][core][INFO] - Step: 24000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:29:39,300][core][INFO] - Step: 26000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:29:55,951][core][INFO] - Step: 28000, Eval mean: 140.2, Eval std: 3.4871191548325386 diff --git a/runs/2025-10-11/23-26-02_agent.use_dueling=true/models/best_model.pt b/runs/2025-10-11/23-26-02_agent.use_dueling=true/models/best_model.pt new file mode 100644 index 0000000..898790f Binary files /dev/null and b/runs/2025-10-11/23-26-02_agent.use_dueling=true/models/best_model.pt differ diff --git a/runs/2025-10-11/23-26-02_agent.use_dueling=true/results.png b/runs/2025-10-11/23-26-02_agent.use_dueling=true/results.png new file mode 100644 index 0000000..c6f6a87 Binary files /dev/null and b/runs/2025-10-11/23-26-02_agent.use_dueling=true/results.png differ diff --git a/runs/2025-10-11/23-38-40_/.hydra/config.yaml b/runs/2025-10-11/23-38-40_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-38-40_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-38-40_/.hydra/hydra.yaml b/runs/2025-10-11/23-38-40_/.hydra/hydra.yaml new file mode 100644 index 0000000..a75607f --- /dev/null +++ b/runs/2025-10-11/23-38-40_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-38-40_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-38-40_/.hydra/overrides.yaml b/runs/2025-10-11/23-38-40_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-38-40_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-38-40_/main.log b/runs/2025-10-11/23-38-40_/main.log new file mode 100644 index 0000000..b24be93 --- /dev/null +++ b/runs/2025-10-11/23-38-40_/main.log @@ -0,0 +1 @@ +[2025-10-11 23:38:41,858][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/23-39-47_/.hydra/config.yaml b/runs/2025-10-11/23-39-47_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-39-47_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-39-47_/.hydra/hydra.yaml b/runs/2025-10-11/23-39-47_/.hydra/hydra.yaml new file mode 100644 index 0000000..57787c6 --- /dev/null +++ b/runs/2025-10-11/23-39-47_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-39-47_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-39-47_/.hydra/overrides.yaml b/runs/2025-10-11/23-39-47_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-39-47_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-39-47_/main.log b/runs/2025-10-11/23-39-47_/main.log new file mode 100644 index 0000000..92583e2 --- /dev/null +++ b/runs/2025-10-11/23-39-47_/main.log @@ -0,0 +1 @@ +[2025-10-11 23:39:49,349][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer diff --git a/runs/2025-10-11/23-41-02_/.hydra/config.yaml b/runs/2025-10-11/23-41-02_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-41-02_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-41-02_/.hydra/hydra.yaml b/runs/2025-10-11/23-41-02_/.hydra/hydra.yaml new file mode 100644 index 0000000..9d613f1 --- /dev/null +++ b/runs/2025-10-11/23-41-02_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-41-02_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-41-02_/.hydra/overrides.yaml b/runs/2025-10-11/23-41-02_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-41-02_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-41-02_/main.log b/runs/2025-10-11/23-41-02_/main.log new file mode 100644 index 0000000..3966e1e --- /dev/null +++ b/runs/2025-10-11/23-41-02_/main.log @@ -0,0 +1,11 @@ +[2025-10-11 23:41:03,998][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 23:41:14,639][core][INFO] - Step: 2000, Eval mean: 262.8, Eval std: 52.92409659125038 +[2025-10-11 23:41:26,221][core][INFO] - Step: 4000, Eval mean: 243.4, Eval std: 65.53655468515262 +[2025-10-11 23:41:37,900][core][INFO] - Step: 6000, Eval mean: 181.4, Eval std: 57.339689570139804 +[2025-10-11 23:41:48,820][core][INFO] - Step: 8000, Eval mean: 98.2, Eval std: 2.638181191654584 +[2025-10-11 23:42:00,404][core][INFO] - Step: 10000, Eval mean: 116.9, Eval std: 5.18555686498567 +[2025-10-11 23:42:12,569][core][INFO] - Step: 12000, Eval mean: 108.0, Eval std: 3.4641016151377544 +[2025-10-11 23:42:24,494][core][INFO] - Step: 14000, Eval mean: 149.4, Eval std: 7.578918128598566 +[2025-10-11 23:42:37,439][core][INFO] - Step: 16000, Eval mean: 250.2, Eval std: 12.416118556135006 +[2025-10-11 23:42:52,336][core][INFO] - Step: 18000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:43:07,082][core][INFO] - Step: 20000, Eval mean: 500.0, Eval std: 0.0 diff --git a/runs/2025-10-11/23-41-02_/models/best_model.pt b/runs/2025-10-11/23-41-02_/models/best_model.pt new file mode 100644 index 0000000..447c587 Binary files /dev/null and b/runs/2025-10-11/23-41-02_/models/best_model.pt differ diff --git a/runs/2025-10-11/23-41-02_/results.png b/runs/2025-10-11/23-41-02_/results.png new file mode 100644 index 0000000..7ba2273 Binary files /dev/null and b/runs/2025-10-11/23-41-02_/results.png differ diff --git a/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..9ce2e03 --- /dev/null +++ b/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..a39835e --- /dev/null +++ b/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-43-22_buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..78c8405 --- /dev/null +++ b/runs/2025-10-11/23-43-22_buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1 @@ +- buffer.use_per=true diff --git a/runs/2025-10-11/23-43-22_buffer.use_per=true/main.log b/runs/2025-10-11/23-43-22_buffer.use_per=true/main.log new file mode 100644 index 0000000..b62152b --- /dev/null +++ b/runs/2025-10-11/23-43-22_buffer.use_per=true/main.log @@ -0,0 +1 @@ +[2025-10-11 23:43:24,112][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and PrioritizedReplayBuffer diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..9ce2e03 --- /dev/null +++ b/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..7be1dcc --- /dev/null +++ b/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-43-52_buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..78c8405 --- /dev/null +++ b/runs/2025-10-11/23-43-52_buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1 @@ +- buffer.use_per=true diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/best_videos.mp4 b/runs/2025-10-11/23-43-52_buffer.use_per=true/best_videos.mp4 new file mode 100644 index 0000000..52d3477 Binary files /dev/null and b/runs/2025-10-11/23-43-52_buffer.use_per=true/best_videos.mp4 differ diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/final_videos.mp4 b/runs/2025-10-11/23-43-52_buffer.use_per=true/final_videos.mp4 new file mode 100644 index 0000000..adc241e Binary files /dev/null and b/runs/2025-10-11/23-43-52_buffer.use_per=true/final_videos.mp4 differ diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/main.log b/runs/2025-10-11/23-43-52_buffer.use_per=true/main.log new file mode 100644 index 0000000..6a77c7a --- /dev/null +++ b/runs/2025-10-11/23-43-52_buffer.use_per=true/main.log @@ -0,0 +1,28 @@ +[2025-10-11 23:43:53,676][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and PrioritizedReplayBuffer +[2025-10-11 23:44:10,340][core][INFO] - Step: 2000, Eval mean: 167.4, Eval std: 44.43467114765226 +[2025-10-11 23:44:28,836][core][INFO] - Step: 4000, Eval mean: 193.3, Eval std: 37.97380676203006 +[2025-10-11 23:44:47,985][core][INFO] - Step: 6000, Eval mean: 100.3, Eval std: 2.7586228448267445 +[2025-10-11 23:45:07,025][core][INFO] - Step: 8000, Eval mean: 110.7, Eval std: 4.050925820105819 +[2025-10-11 23:45:26,143][core][INFO] - Step: 10000, Eval mean: 116.7, Eval std: 3.28785644455472 +[2025-10-11 23:45:45,589][core][INFO] - Step: 12000, Eval mean: 128.9, Eval std: 3.6999999999999997 +[2025-10-11 23:46:04,629][core][INFO] - Step: 14000, Eval mean: 102.4, Eval std: 2.4576411454889016 +[2025-10-11 23:46:24,888][core][INFO] - Step: 16000, Eval mean: 283.4, Eval std: 24.920674148184673 +[2025-10-11 23:46:46,747][core][INFO] - Step: 18000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:47:09,101][core][INFO] - Step: 20000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:47:30,699][core][INFO] - Step: 22000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:47:51,303][core][INFO] - Step: 24000, Eval mean: 142.5, Eval std: 4.5 +[2025-10-11 23:48:14,734][core][INFO] - Step: 26000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:48:37,095][core][INFO] - Step: 28000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:48:59,772][core][INFO] - Step: 30000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:49:20,254][core][INFO] - Step: 32000, Eval mean: 105.8, Eval std: 2.638181191654584 +[2025-10-11 23:49:41,804][core][INFO] - Step: 34000, Eval mean: 290.0, Eval std: 92.10971718553913 +[2025-10-11 23:50:05,661][core][INFO] - Step: 36000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:50:29,141][core][INFO] - Step: 38000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:50:50,699][core][INFO] - Step: 40000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:51:12,136][core][INFO] - Step: 42000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:51:33,089][core][INFO] - Step: 44000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:51:54,989][core][INFO] - Step: 46000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:52:18,438][core][INFO] - Step: 48000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:52:41,758][core][INFO] - Step: 50000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:53:06,483][core][INFO] - Final Eval mean: 500.0, Eval std: 0.0 +[2025-10-11 23:53:12,682][__main__][INFO] - Finish training with eval mean: 500.0 diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/models/best_model.pt b/runs/2025-10-11/23-43-52_buffer.use_per=true/models/best_model.pt new file mode 100644 index 0000000..e799df2 Binary files /dev/null and b/runs/2025-10-11/23-43-52_buffer.use_per=true/models/best_model.pt differ diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/models/final_model.pt b/runs/2025-10-11/23-43-52_buffer.use_per=true/models/final_model.pt new file mode 100644 index 0000000..8519dff Binary files /dev/null and b/runs/2025-10-11/23-43-52_buffer.use_per=true/models/final_model.pt differ diff --git a/runs/2025-10-11/23-43-52_buffer.use_per=true/results.png b/runs/2025-10-11/23-43-52_buffer.use_per=true/results.png new file mode 100644 index 0000000..e0903a6 Binary files /dev/null and b/runs/2025-10-11/23-43-52_buffer.use_per=true/results.png differ diff --git a/runs/2025-10-11/23-55-55_/.hydra/config.yaml b/runs/2025-10-11/23-55-55_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-55-55_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-55-55_/.hydra/hydra.yaml b/runs/2025-10-11/23-55-55_/.hydra/hydra.yaml new file mode 100644 index 0000000..146e140 --- /dev/null +++ b/runs/2025-10-11/23-55-55_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-55-55_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-55-55_/.hydra/overrides.yaml b/runs/2025-10-11/23-55-55_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-55-55_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-55-55_/main.log b/runs/2025-10-11/23-55-55_/main.log new file mode 100644 index 0000000..60ba353 --- /dev/null +++ b/runs/2025-10-11/23-55-55_/main.log @@ -0,0 +1,13 @@ +[2025-10-11 23:55:57,624][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 23:56:07,493][core][INFO] - Step: 2000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 23:56:18,726][core][INFO] - Step: 4000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 23:56:29,929][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 23:56:41,451][core][INFO] - Step: 8000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 23:56:53,228][core][INFO] - Step: 10000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 23:57:05,360][core][INFO] - Step: 12000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 23:57:17,358][core][INFO] - Step: 14000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 23:57:30,126][core][INFO] - Step: 16000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 23:57:43,775][core][INFO] - Step: 18000, Eval mean: 9.6, Eval std: 0.66332495807108 +[2025-10-11 23:57:56,726][core][INFO] - Step: 20000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-11 23:58:09,216][core][INFO] - Step: 22000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-11 23:58:22,288][core][INFO] - Step: 24000, Eval mean: 9.2, Eval std: 0.6 diff --git a/runs/2025-10-11/23-55-55_/models/best_model.pt b/runs/2025-10-11/23-55-55_/models/best_model.pt new file mode 100644 index 0000000..ab8acad Binary files /dev/null and b/runs/2025-10-11/23-55-55_/models/best_model.pt differ diff --git a/runs/2025-10-11/23-55-55_/results.png b/runs/2025-10-11/23-55-55_/results.png new file mode 100644 index 0000000..332fa91 Binary files /dev/null and b/runs/2025-10-11/23-55-55_/results.png differ diff --git a/runs/2025-10-11/23-58-41_/.hydra/config.yaml b/runs/2025-10-11/23-58-41_/.hydra/config.yaml new file mode 100644 index 0000000..2c29bdd --- /dev/null +++ b/runs/2025-10-11/23-58-41_/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 1 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-11/23-58-41_/.hydra/hydra.yaml b/runs/2025-10-11/23-58-41_/.hydra/hydra.yaml new file mode 100644 index 0000000..fbbe31e --- /dev/null +++ b/runs/2025-10-11/23-58-41_/.hydra/hydra.yaml @@ -0,0 +1,154 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: [] + job: + name: main + chdir: true + override_dirname: '' + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-11\23-58-41_ + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-11/23-58-41_/.hydra/overrides.yaml b/runs/2025-10-11/23-58-41_/.hydra/overrides.yaml new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/runs/2025-10-11/23-58-41_/.hydra/overrides.yaml @@ -0,0 +1 @@ +[] diff --git a/runs/2025-10-11/23-58-41_/main.log b/runs/2025-10-11/23-58-41_/main.log new file mode 100644 index 0000000..f339a45 --- /dev/null +++ b/runs/2025-10-11/23-58-41_/main.log @@ -0,0 +1,13 @@ +[2025-10-11 23:58:42,851][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and NormalReplayBuffer +[2025-10-11 23:58:53,927][core][INFO] - Step: 2000, Eval mean: 262.8, Eval std: 52.92409659125038 +[2025-10-11 23:59:05,659][core][INFO] - Step: 4000, Eval mean: 243.4, Eval std: 65.53655468515262 +[2025-10-11 23:59:17,733][core][INFO] - Step: 6000, Eval mean: 181.4, Eval std: 57.339689570139804 +[2025-10-11 23:59:29,144][core][INFO] - Step: 8000, Eval mean: 98.2, Eval std: 2.638181191654584 +[2025-10-11 23:59:41,322][core][INFO] - Step: 10000, Eval mean: 116.9, Eval std: 5.18555686498567 +[2025-10-11 23:59:53,673][core][INFO] - Step: 12000, Eval mean: 108.0, Eval std: 3.4641016151377544 +[2025-10-12 00:00:06,455][core][INFO] - Step: 14000, Eval mean: 149.4, Eval std: 7.578918128598566 +[2025-10-12 00:00:20,513][core][INFO] - Step: 16000, Eval mean: 250.2, Eval std: 12.416118556135006 +[2025-10-12 00:00:35,541][core][INFO] - Step: 18000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:00:51,038][core][INFO] - Step: 20000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:01:06,316][core][INFO] - Step: 22000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:01:21,628][core][INFO] - Step: 24000, Eval mean: 500.0, Eval std: 0.0 diff --git a/runs/2025-10-11/23-58-41_/models/best_model.pt b/runs/2025-10-11/23-58-41_/models/best_model.pt new file mode 100644 index 0000000..447c587 Binary files /dev/null and b/runs/2025-10-11/23-58-41_/models/best_model.pt differ diff --git a/runs/2025-10-11/23-58-41_/results.png b/runs/2025-10-11/23-58-41_/results.png new file mode 100644 index 0000000..9dbfd61 Binary files /dev/null and b/runs/2025-10-11/23-58-41_/results.png differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/config.yaml b/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/config.yaml new file mode 100644 index 0000000..deaf409 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 128 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/hydra.yaml b/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/hydra.yaml new file mode 100644 index 0000000..cbdbc84 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=128 + job: + name: main + chdir: true + override_dirname: buffer.nstep=128 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-04-24_buffer.nstep=128 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/overrides.yaml b/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/overrides.yaml new file mode 100644 index 0000000..1feb3f0 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/.hydra/overrides.yaml @@ -0,0 +1 @@ +- buffer.nstep=128 diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.meta.json new file mode 100644 index 0000000..03b3865 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.meta.json @@ -0,0 +1 @@ +{"step_id": 0, "episode_id": 0, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.mp4 new file mode 100644 index 0000000..3a01d09 Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-0.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.meta.json new file mode 100644 index 0000000..8931a01 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.meta.json @@ -0,0 +1 @@ +{"step_id": 371, "episode_id": 2, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.mp4 new file mode 100644 index 0000000..504eeea Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-2.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-4.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-4.meta.json new file mode 100644 index 0000000..3811355 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/best_videos/eval-episode-4.meta.json @@ -0,0 +1 @@ +{"step_id": 740, "episode_id": 4, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.meta.json new file mode 100644 index 0000000..03b3865 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.meta.json @@ -0,0 +1 @@ +{"step_id": 0, "episode_id": 0, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.mp4 new file mode 100644 index 0000000..7bf3eb2 Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-0.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.meta.json new file mode 100644 index 0000000..f63e88f --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.meta.json @@ -0,0 +1 @@ +{"step_id": 18, "episode_id": 2, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.mp4 new file mode 100644 index 0000000..66a24e6 Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-2.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.meta.json new file mode 100644 index 0000000..33e4f70 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.meta.json @@ -0,0 +1 @@ +{"step_id": 36, "episode_id": 4, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.mp4 new file mode 100644 index 0000000..e5496f5 Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-4.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.meta.json new file mode 100644 index 0000000..baf75d9 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.meta.json @@ -0,0 +1 @@ +{"step_id": 56, "episode_id": 6, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.mp4 new file mode 100644 index 0000000..61b66bd Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-6.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.meta.json b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.meta.json new file mode 100644 index 0000000..967d485 --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.meta.json @@ -0,0 +1 @@ +{"step_id": 75, "episode_id": 8, "content_type": "video/mp4"} \ No newline at end of file diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.mp4 b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.mp4 new file mode 100644 index 0000000..9de6932 Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/final_videos/eval-episode-8.mp4 differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/main.log b/runs/2025-10-12/00-04-24_buffer.nstep=128/main.log new file mode 100644 index 0000000..e5d987a --- /dev/null +++ b/runs/2025-10-12/00-04-24_buffer.nstep=128/main.log @@ -0,0 +1,29 @@ +[2025-10-12 00:04:26,341][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer +[2025-10-12 00:04:35,294][core][INFO] - Step: 2000, Eval mean: 9.3, Eval std: 0.7810249675906655 +[2025-10-12 00:04:46,380][core][INFO] - Step: 4000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-12 00:04:57,225][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-12 00:05:08,588][core][INFO] - Step: 8000, Eval mean: 39.8, Eval std: 15.561490931141527 +[2025-10-12 00:05:20,710][core][INFO] - Step: 10000, Eval mean: 152.7, Eval std: 8.271033792700885 +[2025-10-12 00:05:32,917][core][INFO] - Step: 12000, Eval mean: 107.3, Eval std: 2.934280150224242 +[2025-10-12 00:05:45,065][core][INFO] - Step: 14000, Eval mean: 84.2, Eval std: 3.370459909270544 +[2025-10-12 00:05:57,332][core][INFO] - Step: 16000, Eval mean: 27.0, Eval std: 17.72004514666935 +[2025-10-12 00:06:10,899][core][INFO] - Step: 18000, Eval mean: 184.2, Eval std: 5.035871324805668 +[2025-10-12 00:06:24,745][core][INFO] - Step: 20000, Eval mean: 149.0, Eval std: 14.628738838327793 +[2025-10-12 00:06:37,717][core][INFO] - Step: 22000, Eval mean: 9.7, Eval std: 0.45825756949558394 +[2025-10-12 00:06:50,177][core][INFO] - Step: 24000, Eval mean: 30.1, Eval std: 35.2319457311117 +[2025-10-12 00:07:03,038][core][INFO] - Step: 26000, Eval mean: 53.0, Eval std: 41.6293165929973 +[2025-10-12 00:07:16,211][core][INFO] - Step: 28000, Eval mean: 93.9, Eval std: 4.205948168962618 +[2025-10-12 00:07:28,808][core][INFO] - Step: 30000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-12 00:07:41,971][core][INFO] - Step: 32000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-12 00:07:54,891][core][INFO] - Step: 34000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-12 00:08:08,419][core][INFO] - Step: 36000, Eval mean: 92.3, Eval std: 20.08506908128523 +[2025-10-12 00:08:22,453][core][INFO] - Step: 38000, Eval mean: 108.4, Eval std: 3.2924155266308652 +[2025-10-12 00:08:35,688][core][INFO] - Step: 40000, Eval mean: 9.3, Eval std: 0.45825756949558394 +[2025-10-12 00:08:49,037][core][INFO] - Step: 42000, Eval mean: 10.3, Eval std: 0.6403124237432849 +[2025-10-12 00:09:02,764][core][INFO] - Step: 44000, Eval mean: 55.3, Eval std: 43.91821945388952 +[2025-10-12 00:09:16,935][core][INFO] - Step: 46000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-12 00:09:31,135][core][INFO] - Step: 48000, Eval mean: 66.0, Eval std: 35.608987629529715 +[2025-10-12 00:09:45,275][core][INFO] - Step: 50000, Eval mean: 9.4, Eval std: 0.66332495807108 +[2025-10-12 00:09:48,740][py.warnings][WARNING] - C:\Users\wuzhe\anaconda3\envs\drl_hw2\lib\site-packages\gymnasium\wrappers\monitoring\video_recorder.py:182: UserWarning: WARN: Unable to save last video! Did you call close()? + logger.warn("Unable to save last video! Did you call close()?") + diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/models/best_model.pt b/runs/2025-10-12/00-04-24_buffer.nstep=128/models/best_model.pt new file mode 100644 index 0000000..957516b Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/models/best_model.pt differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/models/final_model.pt b/runs/2025-10-12/00-04-24_buffer.nstep=128/models/final_model.pt new file mode 100644 index 0000000..774eb9d Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/models/final_model.pt differ diff --git a/runs/2025-10-12/00-04-24_buffer.nstep=128/results.png b/runs/2025-10-12/00-04-24_buffer.nstep=128/results.png new file mode 100644 index 0000000..a0b99cd Binary files /dev/null and b/runs/2025-10-12/00-04-24_buffer.nstep=128/results.png differ diff --git a/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/config.yaml b/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/config.yaml new file mode 100644 index 0000000..deaf409 --- /dev/null +++ b/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 128 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/hydra.yaml b/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/hydra.yaml new file mode 100644 index 0000000..2a0f101 --- /dev/null +++ b/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=128 + job: + name: main + chdir: true + override_dirname: buffer.nstep=128 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-13-09_buffer.nstep=128 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/overrides.yaml b/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/overrides.yaml new file mode 100644 index 0000000..1feb3f0 --- /dev/null +++ b/runs/2025-10-12/00-13-09_buffer.nstep=128/.hydra/overrides.yaml @@ -0,0 +1 @@ +- buffer.nstep=128 diff --git a/runs/2025-10-12/00-13-09_buffer.nstep=128/main.log b/runs/2025-10-12/00-13-09_buffer.nstep=128/main.log new file mode 100644 index 0000000..9f3efe5 --- /dev/null +++ b/runs/2025-10-12/00-13-09_buffer.nstep=128/main.log @@ -0,0 +1 @@ +[2025-10-12 00:13:11,183][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer diff --git a/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/config.yaml b/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/config.yaml new file mode 100644 index 0000000..deaf409 --- /dev/null +++ b/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 128 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/hydra.yaml b/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/hydra.yaml new file mode 100644 index 0000000..3e57295 --- /dev/null +++ b/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=128 + job: + name: main + chdir: true + override_dirname: buffer.nstep=128 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-15-00_buffer.nstep=128 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/overrides.yaml b/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/overrides.yaml new file mode 100644 index 0000000..1feb3f0 --- /dev/null +++ b/runs/2025-10-12/00-15-00_buffer.nstep=128/.hydra/overrides.yaml @@ -0,0 +1 @@ +- buffer.nstep=128 diff --git a/runs/2025-10-12/00-15-00_buffer.nstep=128/main.log b/runs/2025-10-12/00-15-00_buffer.nstep=128/main.log new file mode 100644 index 0000000..db50642 --- /dev/null +++ b/runs/2025-10-12/00-15-00_buffer.nstep=128/main.log @@ -0,0 +1 @@ +[2025-10-12 00:15:02,316][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer diff --git a/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/config.yaml b/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/config.yaml new file mode 100644 index 0000000..deaf409 --- /dev/null +++ b/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: false + nstep: 128 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/hydra.yaml b/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/hydra.yaml new file mode 100644 index 0000000..eb06618 --- /dev/null +++ b/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/hydra.yaml @@ -0,0 +1,155 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=128 + job: + name: main + chdir: true + override_dirname: buffer.nstep=128 + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-17-20_buffer.nstep=128 + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/overrides.yaml b/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/overrides.yaml new file mode 100644 index 0000000..1feb3f0 --- /dev/null +++ b/runs/2025-10-12/00-17-20_buffer.nstep=128/.hydra/overrides.yaml @@ -0,0 +1 @@ +- buffer.nstep=128 diff --git a/runs/2025-10-12/00-17-20_buffer.nstep=128/main.log b/runs/2025-10-12/00-17-20_buffer.nstep=128/main.log new file mode 100644 index 0000000..9024ca1 --- /dev/null +++ b/runs/2025-10-12/00-17-20_buffer.nstep=128/main.log @@ -0,0 +1,16 @@ +[2025-10-12 00:17:22,068][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and 128StepReplayBuffer +[2025-10-12 00:17:31,416][core][INFO] - Step: 2000, Eval mean: 9.7, Eval std: 0.7810249675906655 +[2025-10-12 00:17:42,705][core][INFO] - Step: 4000, Eval mean: 10.6, Eval std: 0.66332495807108 +[2025-10-12 00:17:56,752][core][INFO] - Step: 6000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:18:10,715][core][INFO] - Step: 8000, Eval mean: 451.3, Eval std: 146.1 +[2025-10-12 00:18:22,223][core][INFO] - Step: 10000, Eval mean: 9.4, Eval std: 0.4898979485566356 +[2025-10-12 00:18:34,416][core][INFO] - Step: 12000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-12 00:18:49,112][core][INFO] - Step: 14000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:19:04,011][core][INFO] - Step: 16000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:19:16,822][core][INFO] - Step: 18000, Eval mean: 76.1, Eval std: 20.772337374498807 +[2025-10-12 00:19:29,238][core][INFO] - Step: 20000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-12 00:19:45,194][core][INFO] - Step: 22000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:19:58,108][core][INFO] - Step: 24000, Eval mean: 10.4, Eval std: 0.66332495807108 +[2025-10-12 00:20:11,081][core][INFO] - Step: 26000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-12 00:20:26,324][core][INFO] - Step: 28000, Eval mean: 292.1, Eval std: 121.72300522087022 +[2025-10-12 00:20:40,403][core][INFO] - Step: 30000, Eval mean: 37.1, Eval std: 39.459979726299906 diff --git a/runs/2025-10-12/00-17-20_buffer.nstep=128/models/best_model.pt b/runs/2025-10-12/00-17-20_buffer.nstep=128/models/best_model.pt new file mode 100644 index 0000000..efd62a8 Binary files /dev/null and b/runs/2025-10-12/00-17-20_buffer.nstep=128/models/best_model.pt differ diff --git a/runs/2025-10-12/00-17-20_buffer.nstep=128/results.png b/runs/2025-10-12/00-17-20_buffer.nstep=128/results.png new file mode 100644 index 0000000..8661cd5 Binary files /dev/null and b/runs/2025-10-12/00-17-20_buffer.nstep=128/results.png differ diff --git a/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..759f301 --- /dev/null +++ b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 16 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..9c3c9dc --- /dev/null +++ b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,156 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=16 + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.nstep=16,buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-21-15_buffer.nstep=16,buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..51c8568 --- /dev/null +++ b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1,2 @@ +- buffer.nstep=16 +- buffer.use_per=true diff --git a/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/main.log b/runs/2025-10-12/00-21-15_buffer.nstep=16,buffer.use_per=true/main.log new file mode 100644 index 0000000..e69de29 diff --git a/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..759f301 --- /dev/null +++ b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 16 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..13c0e49 --- /dev/null +++ b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,156 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=16 + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.nstep=16,buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-21-37_buffer.nstep=16,buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..51c8568 --- /dev/null +++ b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1,2 @@ +- buffer.nstep=16 +- buffer.use_per=true diff --git a/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/main.log b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/main.log new file mode 100644 index 0000000..da55e9b --- /dev/null +++ b/runs/2025-10-12/00-21-37_buffer.nstep=16,buffer.use_per=true/main.log @@ -0,0 +1 @@ +[2025-10-12 00:21:39,758][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized16StepReplayBuffer diff --git a/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..759f301 --- /dev/null +++ b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 16 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..e97e12e --- /dev/null +++ b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,156 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=16 + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.nstep=16,buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-22-14_buffer.nstep=16,buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..51c8568 --- /dev/null +++ b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1,2 @@ +- buffer.nstep=16 +- buffer.use_per=true diff --git a/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/main.log b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/main.log new file mode 100644 index 0000000..e71a3d9 --- /dev/null +++ b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/main.log @@ -0,0 +1,13 @@ +[2025-10-12 00:22:15,881][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized16StepReplayBuffer +[2025-10-12 00:22:32,257][core][INFO] - Step: 2000, Eval mean: 23.1, Eval std: 10.921996154549772 +[2025-10-12 00:22:50,791][core][INFO] - Step: 4000, Eval mean: 10.3, Eval std: 1.2688577540449522 +[2025-10-12 00:23:08,418][core][INFO] - Step: 6000, Eval mean: 13.5, Eval std: 2.247220505424423 +[2025-10-12 00:23:26,620][core][INFO] - Step: 8000, Eval mean: 13.0, Eval std: 1.2649110640673518 +[2025-10-12 00:23:44,856][core][INFO] - Step: 10000, Eval mean: 12.3, Eval std: 0.9 +[2025-10-12 00:24:03,104][core][INFO] - Step: 12000, Eval mean: 13.0, Eval std: 1.7888543819998317 +[2025-10-12 00:24:21,653][core][INFO] - Step: 14000, Eval mean: 11.1, Eval std: 3.1448370387032774 +[2025-10-12 00:24:40,292][core][INFO] - Step: 16000, Eval mean: 14.6, Eval std: 3.6386810797320503 +[2025-10-12 00:24:58,872][core][INFO] - Step: 18000, Eval mean: 13.9, Eval std: 1.6401219466856727 +[2025-10-12 00:25:18,000][core][INFO] - Step: 20000, Eval mean: 26.4, Eval std: 13.462540622037135 +[2025-10-12 00:25:36,671][core][INFO] - Step: 22000, Eval mean: 42.0, Eval std: 23.112767034693185 +[2025-10-12 00:25:54,993][core][INFO] - Step: 24000, Eval mean: 12.8, Eval std: 1.6613247725836149 diff --git a/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/models/best_model.pt b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/models/best_model.pt new file mode 100644 index 0000000..4dbe3ce Binary files /dev/null and b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/models/best_model.pt differ diff --git a/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/results.png b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/results.png new file mode 100644 index 0000000..6ae70db Binary files /dev/null and b/runs/2025-10-12/00-22-14_buffer.nstep=16,buffer.use_per=true/results.png differ diff --git a/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..20f0208 --- /dev/null +++ b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 10 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..24871e0 --- /dev/null +++ b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,156 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=10 + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.nstep=10,buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-26-19_buffer.nstep=10,buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..4884fa7 --- /dev/null +++ b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1,2 @@ +- buffer.nstep=10 +- buffer.use_per=true diff --git a/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/main.log b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/main.log new file mode 100644 index 0000000..a3712ce --- /dev/null +++ b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/main.log @@ -0,0 +1,25 @@ +[2025-10-12 00:26:21,193][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized10StepReplayBuffer +[2025-10-12 00:26:37,205][core][INFO] - Step: 2000, Eval mean: 12.2, Eval std: 0.8717797887081347 +[2025-10-12 00:26:54,645][core][INFO] - Step: 4000, Eval mean: 9.6, Eval std: 1.2806248474865698 +[2025-10-12 00:27:12,056][core][INFO] - Step: 6000, Eval mean: 9.2, Eval std: 0.6 +[2025-10-12 00:27:29,842][core][INFO] - Step: 8000, Eval mean: 21.6, Eval std: 5.730619512757761 +[2025-10-12 00:27:48,840][core][INFO] - Step: 10000, Eval mean: 227.8, Eval std: 182.14543639630392 +[2025-10-12 00:28:09,044][core][INFO] - Step: 12000, Eval mean: 432.6, Eval std: 135.09641001891944 +[2025-10-12 00:28:29,323][core][INFO] - Step: 14000, Eval mean: 360.9, Eval std: 121.26866866590068 +[2025-10-12 00:28:49,316][core][INFO] - Step: 16000, Eval mean: 368.7, Eval std: 105.315763302556 +[2025-10-12 00:29:09,905][core][INFO] - Step: 18000, Eval mean: 370.6, Eval std: 114.3268997218065 +[2025-10-12 00:29:30,078][core][INFO] - Step: 20000, Eval mean: 380.4, Eval std: 108.37545847653887 +[2025-10-12 00:29:50,352][core][INFO] - Step: 22000, Eval mean: 436.0, Eval std: 89.43489251964247 +[2025-10-12 00:30:10,478][core][INFO] - Step: 24000, Eval mean: 437.1, Eval std: 93.34286260877154 +[2025-10-12 00:30:30,447][core][INFO] - Step: 26000, Eval mean: 451.8, Eval std: 83.40119903214821 +[2025-10-12 00:30:50,912][core][INFO] - Step: 28000, Eval mean: 439.1, Eval std: 85.54115968351142 +[2025-10-12 00:31:11,646][core][INFO] - Step: 30000, Eval mean: 454.0, Eval std: 69.80257874892588 +[2025-10-12 00:31:32,031][core][INFO] - Step: 32000, Eval mean: 423.2, Eval std: 75.24333857558422 +[2025-10-12 00:31:52,381][core][INFO] - Step: 34000, Eval mean: 386.5, Eval std: 103.64386137152552 +[2025-10-12 00:32:12,858][core][INFO] - Step: 36000, Eval mean: 456.8, Eval std: 84.32057874564192 +[2025-10-12 00:32:33,233][core][INFO] - Step: 38000, Eval mean: 448.7, Eval std: 96.5691979877642 +[2025-10-12 00:32:53,194][core][INFO] - Step: 40000, Eval mean: 455.4, Eval std: 83.44123680770797 +[2025-10-12 00:33:13,862][core][INFO] - Step: 42000, Eval mean: 431.9, Eval std: 89.02634441557173 +[2025-10-12 00:33:34,167][core][INFO] - Step: 44000, Eval mean: 426.1, Eval std: 91.27370924861113 +[2025-10-12 00:33:54,320][core][INFO] - Step: 46000, Eval mean: 422.8, Eval std: 84.57635603405954 +[2025-10-12 00:34:14,480][core][INFO] - Step: 48000, Eval mean: 439.7, Eval std: 94.10850121003946 diff --git a/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/models/best_model.pt b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/models/best_model.pt new file mode 100644 index 0000000..51df7de Binary files /dev/null and b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/models/best_model.pt differ diff --git a/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/results.png b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/results.png new file mode 100644 index 0000000..6b06f45 Binary files /dev/null and b/runs/2025-10-12/00-26-19_buffer.nstep=10,buffer.use_per=true/results.png differ diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/config.yaml b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/config.yaml new file mode 100644 index 0000000..7069b46 --- /dev/null +++ b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/config.yaml @@ -0,0 +1,33 @@ +seed: 42 +env_name: CartPole-v1 +train: + nstep: ${buffer.nstep} + timesteps: 50000 + batch_size: 128 + test_every: 2500 + eps_max: 1 + eps_min: 0.05 + eps_steps: 12500 + start_steps: 0 + plot_interval: 2000 + eval_interval: 2000 + eval_episodes: 10 +agent: + gamma: 0.99 + lr: 0.002 + tau: 0.1 + nstep: ${buffer.nstep} + target_update_interval: 3 + hidden_size: 64 + activation: + _target_: torch.nn.ELU + use_dueling: false + use_double: false +buffer: + capacity: 50000 + use_per: true + nstep: 5 + gamma: ${agent.gamma} + per_alpha: 0.7 + per_beta: 0.4 + per_eps: 0.01 diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/hydra.yaml b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/hydra.yaml new file mode 100644 index 0000000..c8f200e --- /dev/null +++ b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/hydra.yaml @@ -0,0 +1,156 @@ +hydra: + run: + dir: ./runs/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + sweep: + dir: ./sweeps/${now:%Y-%m-%d}/${now:%H-%M-%S}_${hydra.job.override_dirname} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - buffer.nstep=5 + - buffer.use_per=true + job: + name: main + chdir: true + override_dirname: buffer.nstep=5,buffer.use_per=true + id: ??? + num: ??? + config_name: config + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.3' + cwd: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2 + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\hw2\cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: D:\Documents\Nextcloud\Documents\Project WUSTL\Academic\2025_Fall\CSE5100\Homeworks\hw2\runs\2025-10-12\00-34-29_buffer.nstep=5,buffer.use_per=true + choices: + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/overrides.yaml b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/overrides.yaml new file mode 100644 index 0000000..61cfb6d --- /dev/null +++ b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/.hydra/overrides.yaml @@ -0,0 +1,2 @@ +- buffer.nstep=5 +- buffer.use_per=true diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/best_videos.mp4 b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/best_videos.mp4 new file mode 100644 index 0000000..26824b6 Binary files /dev/null and b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/best_videos.mp4 differ diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/final_videos.mp4 b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/final_videos.mp4 new file mode 100644 index 0000000..14dead2 Binary files /dev/null and b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/final_videos.mp4 differ diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/main.log b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/main.log new file mode 100644 index 0000000..a53d42a --- /dev/null +++ b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/main.log @@ -0,0 +1,28 @@ +[2025-10-12 00:34:31,512][__main__][INFO] - Training for 50000 timesteps with NormalQNetwork and Prioritized5StepReplayBuffer +[2025-10-12 00:34:50,453][core][INFO] - Step: 2000, Eval mean: 442.7, Eval std: 115.3568810257975 +[2025-10-12 00:35:10,869][core][INFO] - Step: 4000, Eval mean: 449.6, Eval std: 101.87462883367968 +[2025-10-12 00:35:30,844][core][INFO] - Step: 6000, Eval mean: 367.7, Eval std: 101.56972974267482 +[2025-10-12 00:35:50,826][core][INFO] - Step: 8000, Eval mean: 284.6, Eval std: 37.296648643008126 +[2025-10-12 00:36:10,180][core][INFO] - Step: 10000, Eval mean: 224.7, Eval std: 18.10552401892859 +[2025-10-12 00:36:30,662][core][INFO] - Step: 12000, Eval mean: 317.9, Eval std: 68.10058736897943 +[2025-10-12 00:36:49,780][core][INFO] - Step: 14000, Eval mean: 156.4, Eval std: 7.255342858886822 +[2025-10-12 00:37:09,606][core][INFO] - Step: 16000, Eval mean: 213.9, Eval std: 15.286922515666781 +[2025-10-12 00:37:31,528][core][INFO] - Step: 18000, Eval mean: 489.4, Eval std: 31.8 +[2025-10-12 00:37:51,012][core][INFO] - Step: 20000, Eval mean: 148.8, Eval std: 6.446704584514478 +[2025-10-12 00:38:12,456][core][INFO] - Step: 22000, Eval mean: 475.1, Eval std: 74.7 +[2025-10-12 00:38:31,795][core][INFO] - Step: 24000, Eval mean: 156.9, Eval std: 9.235258523723092 +[2025-10-12 00:38:51,389][core][INFO] - Step: 26000, Eval mean: 201.0, Eval std: 9.02219485491197 +[2025-10-12 00:39:12,304][core][INFO] - Step: 28000, Eval mean: 487.5, Eval std: 37.5 +[2025-10-12 00:39:33,586][core][INFO] - Step: 30000, Eval mean: 475.4, Eval std: 73.8 +[2025-10-12 00:39:54,512][core][INFO] - Step: 32000, Eval mean: 486.2, Eval std: 41.4 +[2025-10-12 00:40:13,757][core][INFO] - Step: 34000, Eval mean: 129.1, Eval std: 6.122907805936653 +[2025-10-12 00:40:35,414][core][INFO] - Step: 36000, Eval mean: 488.8, Eval std: 33.6 +[2025-10-12 00:40:57,324][core][INFO] - Step: 38000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:41:16,949][core][INFO] - Step: 40000, Eval mean: 225.3, Eval std: 20.459960899278375 +[2025-10-12 00:41:37,212][core][INFO] - Step: 42000, Eval mean: 209.4, Eval std: 8.321057625085889 +[2025-10-12 00:41:57,108][core][INFO] - Step: 44000, Eval mean: 204.2, Eval std: 10.934349546269315 +[2025-10-12 00:42:18,230][core][INFO] - Step: 46000, Eval mean: 470.0, Eval std: 60.13484846576068 +[2025-10-12 00:42:38,199][core][INFO] - Step: 48000, Eval mean: 181.5, Eval std: 11.351211389098522 +[2025-10-12 00:43:00,221][core][INFO] - Step: 50000, Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:43:23,166][core][INFO] - Final Eval mean: 500.0, Eval std: 0.0 +[2025-10-12 00:43:29,422][__main__][INFO] - Finish training with eval mean: 500.0 diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/models/best_model.pt b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/models/best_model.pt new file mode 100644 index 0000000..b710939 Binary files /dev/null and b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/models/best_model.pt differ diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/models/final_model.pt b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/models/final_model.pt new file mode 100644 index 0000000..a97536e Binary files /dev/null and b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/models/final_model.pt differ diff --git a/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/results.png b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/results.png new file mode 100644 index 0000000..a2588d9 Binary files /dev/null and b/runs/2025-10-12/00-34-29_buffer.nstep=5,buffer.use_per=true/results.png differ