diff --git a/pages/CSE559A/CSE559A_L1.md b/pages/CSE559A/CSE559A_L1.md index 9ad90d9..a8d6c78 100644 --- a/pages/CSE559A/CSE559A_L1.md +++ b/pages/CSE559A/CSE559A_L1.md @@ -1,4 +1,4 @@ -# Lecture 1 +# CSE559A Lecture 1 ## Introducing the syllabus diff --git a/pages/CSE559A/CSE559A_L2.md b/pages/CSE559A/CSE559A_L2.md index 532b204..462451b 100644 --- a/pages/CSE559A/CSE559A_L2.md +++ b/pages/CSE559A/CSE559A_L2.md @@ -1,4 +1,4 @@ -# Lecture 2 +# CSE559A Lecture 2 ## The Geometry of Image Formation diff --git a/pages/CSE559A/CSE559A_L3.md b/pages/CSE559A/CSE559A_L3.md index 431e696..2f0a2e6 100644 --- a/pages/CSE559A/CSE559A_L3.md +++ b/pages/CSE559A/CSE559A_L3.md @@ -1,4 +1,4 @@ -# Lecture 3 +# CSE559A Lecture 3 ## Image formation diff --git a/pages/CSE559A/CSE559A_L4.md b/pages/CSE559A/CSE559A_L4.md index 69567c4..ebc98a0 100644 --- a/pages/CSE559A/CSE559A_L4.md +++ b/pages/CSE559A/CSE559A_L4.md @@ -1,4 +1,4 @@ -# Lecture 4 +# CSE559A Lecture 4 ## Practical issues with filtering diff --git a/pages/CSE559A/CSE559A_L5.md b/pages/CSE559A/CSE559A_L5.md index f925461..9ff86ae 100644 --- a/pages/CSE559A/CSE559A_L5.md +++ b/pages/CSE559A/CSE559A_L5.md @@ -1,4 +1,4 @@ -# Lecture 5 +# CSE559A Lecture 5 ## Continue on linear interpolation diff --git a/pages/CSE559A/CSE559A_L6.md b/pages/CSE559A/CSE559A_L6.md index 44f1565..41e8322 100644 --- a/pages/CSE559A/CSE559A_L6.md +++ b/pages/CSE559A/CSE559A_L6.md @@ -1,4 +1,4 @@ -# Lecture 6 +# CSE559A Lecture 6 ## Continue on Light, eye/camera, and color diff --git a/pages/CSE559A/CSE559A_L7.md b/pages/CSE559A/CSE559A_L7.md index 252ad04..faa94c4 100644 --- a/pages/CSE559A/CSE559A_L7.md +++ b/pages/CSE559A/CSE559A_L7.md @@ -1,4 +1,4 @@ -# Lecture 7 +# CSE559A Lecture 7 ## Computer Vision (In Artificial Neural Networks for Image Understanding) diff --git a/pages/CSE559A/CSE559A_L8.md b/pages/CSE559A/CSE559A_L8.md index 93b26f7..cbb9906 100644 --- a/pages/CSE559A/CSE559A_L8.md +++ b/pages/CSE559A/CSE559A_L8.md @@ -1,4 +1,4 @@ -# Lecture 8 +# CSE559A Lecture 8 Paper review sharing. @@ -78,10 +78,3 @@ $$ $$ #### General backpropagation algorithm - - - - - - - diff --git a/pages/CSE559A/CSE559A_L9.md b/pages/CSE559A/CSE559A_L9.md new file mode 100644 index 0000000..2eb9594 --- /dev/null +++ b/pages/CSE559A/CSE559A_L9.md @@ -0,0 +1,102 @@ +# CSE559A Lecture 9 + +## Continue on ML for computer vision + +### Backpropagation + +#### Computation graphs + +SGD update for each parameter + +$$ +w_k\gets w_k-\eta\frac{\partial e}{\partial w_k} +$$ + +$e$ is the error function. + +#### Using the chain rule + +Suppose $k=1$, $e=l(f_1(x,w_1),y)$ + +Example: $e=(f_1(x,w_1)-y)^2$ + +So $h_1=f_1(x,w_1)=w^T_1x$, $e=l(h_1,y)=(y-h_1)^2$ + +$$ +\frac{\partial e}{\partial w_1}=\frac{\partial e}{\partial h_1}\frac{\partial h_1}{\partial w_1} +$$ + +$$ +\frac{\partial e}{\partial h_1}=2(h_1-y) +$$ + +$$ +\frac{\partial h_1}{\partial w_1}=x +$$ + +$$ +\frac{\partial e}{\partial w_1}=2(h_1-y)x +$$ + +For the general cases, + +$$ +\frac{\partial e}{\partial w_k}=\frac{\partial e}{\partial h_K}\frac{\partial h_K}{\partial h_{K-1}}\cdots\frac{\partial h_{k+2}}{\partial h_{k+1}}\frac{\partial h_{k+1}}{\partial h_k}\frac{\partial h_k}{\partial w_k} +$$ + +Where the upstream gradient $\frac{\partial e}{\partial h_K}$ is known, and the local gradient $\frac{\partial h_k}{\partial w_k}$ is known. + +#### General backpropagation algorithm + +The adding layer is the gradient distributor layer. +The multiplying layer is the gradient switcher layer. +The max operation is the gradient router layer. + +![Images of propagation](https://static.notenextra.trance-0.com/CSE559A/General_computation_graphs_for_MLP.png) + +Simple example: Element-wise operation (ReLU) + +$f(x)=ReLU(x)=max(0,x)$ + +$$ +\frac{\partial z}{\partial x}=\begin{pmatrix} +\frac{\partial z_1}{\partial x_1} & 0 & \cdots & 0 \\ +0 & \frac{\partial z_2}{\partial x_2} & \cdots & 0 \\ +\vdots & \vdots & \ddots & \vdots \\ +0 & 0 & \cdots & \frac{\partial z_n}{\partial x_n} +\end{pmatrix} +$$ + +Where $\frac{\partial z_i}{\partial x_j}=1$ if $i=j$ and $z_i>0$, otherwise $\frac{\partial z_i}{\partial x_j}=0$. + +When $\forall x_i<0$ then $\frac{\partial z}{\partial x}=0$ (dead ReLU) + +Other examples on ppt. + +## Convolutional Neural Networks + +### Basic Convolutional layer + +#### Flatten layer + +Fully connected layer, operate on vectorized image. + +With the multi-layer perceptron, the neural network trying to fit the templates. + +![Flatten layer](https://static.notenextra.trance-0.com/CSE559A/Flatten_layer.png) + +#### Convolutional layer + +Limit the receptive fields of units, tiles them over the input image, and share the weights. + +Equivalent to sliding the learned filter over the image , computing dot products at each location. + +![Convolutional layer](https://static.notenextra.trance-0.com/CSE559A/Convolutional_layer.png) + +Padding: Add a border of zeros around the image. (higher padding, larger output size) + +Stride: The step size of the filter. (higher stride, smaller output size) + +### Variants 1x1 convolutions, depthwise convolutions + +### Backward pass diff --git a/pages/CSE559A/_meta.js b/pages/CSE559A/_meta.js index ed50190..b79d0b8 100644 --- a/pages/CSE559A/_meta.js +++ b/pages/CSE559A/_meta.js @@ -11,4 +11,5 @@ export default { CSE559A_L6: "Computer Vision (Lecture 6)", CSE559A_L7: "Computer Vision (Lecture 7)", CSE559A_L8: "Computer Vision (Lecture 8)", + CSE559A_L9: "Computer Vision (Lecture 9)", } diff --git a/pages/Math416/Math416_L1.md b/pages/Math416/Math416_L1.md index 6163d5f..6597f2b 100644 --- a/pages/Math416/Math416_L1.md +++ b/pages/Math416/Math416_L1.md @@ -1,4 +1,4 @@ -# Lecture 1 +# Math416 Lecture 1 ## Chapter 1: Complex Numbers diff --git a/pages/Math416/Math416_L2.md b/pages/Math416/Math416_L2.md index 2246204..5bdc07b 100644 --- a/pages/Math416/Math416_L2.md +++ b/pages/Math416/Math416_L2.md @@ -1,4 +1,4 @@ -# Lecture 2 +# Math416 Lecture 2 ## Review? diff --git a/public/CSE559A/Convolutional_layer.png b/public/CSE559A/Convolutional_layer.png new file mode 100644 index 0000000..bed45e4 Binary files /dev/null and b/public/CSE559A/Convolutional_layer.png differ diff --git a/public/CSE559A/Flatten_layer.png b/public/CSE559A/Flatten_layer.png new file mode 100644 index 0000000..1e3c109 Binary files /dev/null and b/public/CSE559A/Flatten_layer.png differ diff --git a/public/CSE559A/General_computation_graphs_for_MLP.png b/public/CSE559A/General_computation_graphs_for_MLP.png new file mode 100644 index 0000000..ecadbd2 Binary files /dev/null and b/public/CSE559A/General_computation_graphs_for_MLP.png differ