Ch05_GeometryOfVectorSpaces.tex

\chapter{Geometry in Vector Spaces}
Vector spaces give a rich environment to explore the relationships between 
objects and gives natural abstractions to other types of {\it vectors} (e.g. functions,
matrices, polynimials, etc).  Most of the study of vector spaces involves abstraction
and making precise ideas that spanned many decades, or centuries, in many different fields
of science and mathematics.  In this sense, the study of vector spaces leans heavily on
past mathematical results and seeks to give them order and structure.  You
have seen that the study of vector spaces really acts as a unifying language to talk about
a wide variety of mathematical objects.

In this chapter we layer the language of abstract vector spaces and the language of
geometry.  In particular, we will build and abstract the familiar ideas of angle,
othogonality, lengths of vectors, and distance in general vector spaces.  We will make
precise the ideas of the {\it size of a function} or the {\it distance between two
matrices} or {\it polynomials}.  These notions are likely second nature in our familiar
vector spaces 
$\mathbb{R}^2$ and $\mathbb{R}^3$, but what about in spaces of matrices? spaces of
functions? spaces of polynomials?  The notions of angle and  distance can be abstracted in
a natural and beautiful way so that our intuitive ideas still hold, but we also get
something mathematically meaningful in the more abstract spaces.  We'll start with an idea
that is probably familiar to you: the dot product\footnote{If you haven't had
multivariable calculus or calculus-based physics then maybe the idea of a dot product will
be new to you.  Don't worry.  We'll introduce everything from scratch here.}.  From the
dot product we will build a similar idea is more general vector spaces so that the ideas
of angle and perpendicularity come along for the ride.  Here we go!

\section{The Geometry of Euclidean Space}
At this point we have talked almost exclusively about linear combinations and the spaces
associated with them.  We have not, however, discussed the geometry of vector spaces.  So
far we haven't generalized the notions of angle and length of vectors to our large view of
vector spaces.  You may recall things like projections, angles, norms (lengths) from
$\mathbb{R}^2$ or $\mathbb{R}^3$ as discussed in physics or in multivariable calculus but
you need to keep in mind that this is only a limited view of the world of vector spaces.
Let's jump right in by filling in some definitions and theorems that you likely already
know

\subsection{The Dot Product}
The following are two different familiar definitions of the dot product.  The first gives
a purely algebraic formula for the dot product and the second gives a more geometric
definition.  They are indeed equivalent definitions.
\begin{definition}[Algebraic Dot Product in $\mathbb{R}^n$]
    Let $\bu, \bv \in \mathbb{R}^n$.  The {\bf dot product} of $\bu$ and $\bv$ is defined
    algebraically as
    \[ \bu \cdot \bv = u_1 v_1 + u_2 v_2 + \cdots + u_n v_n = \sum_{j=1}^n u_j v_j. \]
    Notice that this definition doesn't explicitly mention the angle between the vectors.
\end{definition}
% \solution{
%     $\bu \cdot \bv = \sum_{j=1}^n u_j v_j$
% }

\begin{definition}[Geometric Dot Product in $\mathbb{R}^n$]
    If $\bu,\bv\in\mathbb{R}^n$ then the relationship between the dot product of the
    vectors and the angle between the vectors is
    \[ \bu \cdot \bv = \|\bu\| \|\bv\| \cos \theta. \]
    Notice that if you want the angle between two vectors then 
    \[ \theta = \cos^{-1} \left( \frac{\bu \cdot \bv}{\| \bu \| \|\bv\|} \right). \]
\end{definition}
One should note that the angle formula only holds if both $\|\bu\|\ne0$ and $\|\bv\|\ne0$.

Now we build the mathematical notation for orthogonality and length of vectors in
$\mathbb{R}^n$.  Both of these geometric concepts are build upon the dot product.
\begin{definition}[Orthogonal Vectors]
    Two vectors are said to be orthogonal (perpendicular) if their dot product is zero.
\end{definition}
\begin{problem}
    Use the geometric definition of the dot product to prove that statements 
    \begin{itemize}
        \item ``the dot product of two vectors is zero'', and 
        \item ``the vectors are perpendicular'' 
    \end{itemize}
    are indeed the same based on your familiar understanding of what ``perpendicular''
    means.
\end{problem}
% \begin{thm}[Orthogonal Vectors in $\mathbb{R}^n$]
%     If $\bu,\bv\in\mathbb{R}^n$ then $\bu$ is orthogonal (perpendicular) to $\bv$ if and
%     only if \underline{\hspace{1in}}.
% \end{thm}
% \solution{
% $\bu \cdot \bv = 0$
% }
% \begin{proof}
%     (prove this theorem)
% \end{proof}
\solution{
$\bu \cdot \bv = \|\bv \| \|\bu\| \cos \theta$ and if $\theta = \pi/2$ we know that
$\cos \theta = 0$.  Hence $\bu \cdot \bv = 0$.
}

\begin{definition}[Length of Vectors in $\mathbb{R}^n$]
    Let $\bu \in \mathbb{R}^n$.  The {\bf length (norm)} of $\bu$ is
    \[ \| \bu \| = \sqrt{\bu \cdot \bu} = \underline{\hspace{2in}} \]
    (fill in the blank using the algebraic definition of the dot product)
\end{definition}
\solution{
    $\|\bu\| = \sum_{j=1}^n u_j^2$
}

\begin{problem}
    If $\bu,\bv \in \mathbb{R}^2$ then what familiar theorem do you see in the definition
    of the length of a vector?  Another way to put this is: the definition of the length
    of a vector in $\mathbb{R}^n$ is a generalization of what familiar theorem?
\end{problem}
\solution{
    This is just Pythagorean Theorem.
}

\begin{definition}[Distance Between Vectors in $\mathbb{R}^n$]
    Let $\bu,\bv\in\mathbb{R}^n$.  The {\bf distance between} $\bu$ and $\bv$ is
    \[ \text{dist}(\bu,\bv) = \| \bu - \bv\| = \underline{\hspace{1in}} \]
    (fill in the blank)
\end{definition}
\solution{
    $\text{dist}(\bu,\bv) = \|\bu-\bv\|$
}

\begin{definition}[Unit Vectors in $\mathbb{R}^n$]
    Let $\bv \in \mathbb{R}^n$.  We say that $\bv$ is a unit vector if $\|\bv\| = 1$.  If
    you have a vector $\bv$ that is not a unit vector (has length other than 1) then you
    can scale it to become a unit vector by dividing by its length
    \[ (\text{unit vector in the direction of $\bv$}) = \bu = \frac{\bv}{\|\bv\|}. \]
\end{definition}

OK.  Most of the geometric players are on the table: angle, distance, length, and
perpendicularity.  Here we give a straight forward example showing how to use these ideas
in $\mathbb{R}^3$.


\begin{problem}
    Let $\bu$ and $\bv$ be defined as.  Find $\bu \cdot \bv$, $\|\bu\|$, $\|\bv\|$, and
    the angle between them.
    \[ \bu = \left( \begin{array}{c} 0 \\ 1 \\ -1 \end{array} \right) \]
    \[ \bv = \left( \begin{array}{c} 4 \\ 2 \\ -3 \end{array} \right) \]
% 
% \begin{enumerate}
%     \item[(a)] 
% $\left( \begin{array}{c} 0 \\ 2 \\ 3 \end{array} \right)$
% \item[(b)] 5
% \item[(c)] 0
% \item[(d)] The dot product cannot be computed for these vectors.
% \end{enumerate}
\end{problem}
% \begin{problem}
%     \begin{itemize}
%             \input{ClickerQuestions/LA.00.22.010}
%     \end{itemize}
% \end{problem}
\solution{
The dot product is $(0)(4) + (1)(2) + (-1)(-3) = 2+3 = 5$
}
\begin{example}
    Let $\bu, \bv \in \mathbb{R}^3$ be defined as
    \[ \bu = \begin{pmatrix} 2 \\ -1 \\ 4 \end{pmatrix} \quad \text{and} \quad \bv =
    \begin{pmatrix} 0 \\ 9 \\ -3 \end{pmatrix} \]
    What are $\bu \cdot \bv$, $\|\bu\|$, $\|\bv\|$, and $\text{dist}(\bu,\bv)$?  Are the
    two vectors orthogonal?   What is the angle between them? \\ {\bf
    Solution:} 
    \begin{flalign*}
        \bu \cdot \bv &= (2)(0) + (-1)(9) + (4)(-3) = -9 - 12 = -21 \\
        \|\bu\| &= \sqrt{2^2 + (-1)^1 + 4^2} = \sqrt{4 + 1 + 16} = \sqrt{21} \\
        \|\bv\| &= \sqrt{0^2 + 9^2 + (-3)^2} = \sqrt{81 + 9} = \sqrt{90} \\
        \text{dist}(\bu,\bv) &= \| \bu - \bv\| = \| \begin{pmatrix} 2 \\ -10 \\ 7
        \end{pmatrix} \| = \sqrt{4+100+49} = \sqrt{153}.
    \end{flalign*}
    The vectors are not orthogonal since $\bu \cdot \bv \ne 0$.  The angle between the
    vectors is 
    \[ \theta = \cos^{-1} \left( \frac{\bu \cdot \bv}{\|\bu\| \|\bv\|} \right) =
    \cos^{-1}\left( \frac{-21}{\sqrt{21}\sqrt{90}} \right) \]
\end{example}


\subsection{Projections}
Finally we are going to discuss projections.  When dealing with projections you should be
thinking about how shadows are cast between vectors.  To solidify this notion (even though
you likely already know it) let's look at some projections in $\mathbb{R}^2$ before we
ramp up the dimension.  Take a look at Figure \ref{fig:proj_R2}.  We would like to project
vector $\bu$ onto vector $\bv$ and by that we mean that we would like to draw a vector
(depicted by the dashed vector $\bw$ in the figure) that is perpendicular to $\bv$ and meets the
head of $\bu$. This projection creates the vector $\hat{\bv}$ so that $\hat{\bv}$ points
in exactly the same direction as $\bv$ but $\hat{\bv} \perp \bw$. Since $\bv$ and
$\hat{\bv}$ point in the same direction we know that $\hat{\bv} = c\bv$ for some scalar
$c$.  Furthermore, we know that $\bw + \hat{\bv} = \bu$ so $\bw = \bu - \hat{\bv}$. Therefore,
\begin{flalign*}
    &0 = \hat{\bv} \cdot \bw = c\bv \cdot \left( \bu - c\bv \right) \\
    &\implies c\bv \cdot \bu - c^2 \bv \cdot \bv = 0 \\
    &\implies \bu \cdot \bv = c \bv \cdot \bv \\
    &\implies c = \frac{\bu \cdot \bv}{\bv \cdot \bv}
\end{flalign*}
\begin{figure}[ht!]
    \begin{center}
        \begin{tikzpicture}
            \draw[->,thick, blue] (0,0) -- (3,1) node[anchor=west]{$\bv$};
            \draw[->,thick, red] (0,0) -- (1,3) node[anchor=south]{$\bu$};
            \draw[->,thick, dashed, black] (1.8,0.6) -- (1,3);
            \draw[black] (1.4,2) node[anchor=west]{$\bw$};
            \draw[->,thick, dashed, blue] (0.1,-0.1) -- (1.9,0.5);
            \draw[blue] (1,0.25) node[anchor=north]{$\hat{\bv}$};
        \end{tikzpicture}
    \end{center}
    \caption{Depiction of vector projection in $\mathbb{R}^2$.}
    \label{fig:proj_R2}
\end{figure}

All of the prior discuss proves the following theorem but notice that we never made any
mention explicitly about the vectors living in $\mathbb{R}^2$.  In fact, the proof that we
gave works generally in $\mathbb{R}^n$.
\begin{thm}[Orthogonal Projection]
    Let $\bu, \bv \in \mathbb{R}^n$.  If we are to project $\bu$ onto $\bv$ as in Figure
    \ref{fig:proj_R2} we get
    \begin{flalign*}
        \text{proj}_{\bv}(\bu) &= \hat{\bv} = \left( \frac{\bv \cdot \bu}{\bv \cdot \bv}
        \right) \bv =
        \text{projection of $\bu$ onto $\bv$} \\
        \bw &= \bu - \hat{\bv} = \bu - \left( \frac{\bv \cdot \bu}{\bv \cdot \bv} \right)
        \bv
        = \text{projection error}
    \end{flalign*}
    The vector $\bw$ is often called the {\it error} in the projection.
\end{thm}



\begin{problem}
    If $\bb = \left( \begin{array}{c} 3 \\ -1  \end{array} \right)$ and $y = \left(
    \begin{array}{c} 2 \\ 1  \end{array} \right),$ then what is the orthogonal projection of $\bb$
    onto $\by$?  Find your solution analytically and draw of graph depicting your answer.
% 
% \begin{enumerate}
%     \item[(a)] $\left( \begin{array}{c} 2 \\ 1  \end{array} \right)$
%     \item[(b)] $\left( \begin{array}{c} 3/2 \\ -1/2 \end{array} \right)$
%     \item[(c)] $\left( \begin{array}{c} 10 \\ 5  \end{array} \right)$
%     \item[(d)] $\left( \begin{array}{c} 1/10 \\ 3/10  \end{array} \right)$
% \end{enumerate}
% 
\end{problem}
% \begin{problem}
%     \begin{itemize}
%             \input{ClickerQuestions/LA.00.24.010}
%     \end{itemize}
% \end{problem}
\solution{
    $\text{proj}_y b = \left( \frac{b \cdot y}{y \dot y} \right) y = \left(
    \frac{5}{5}
    \right) \begin{pmatrix} 2 \\ 1 \end{pmatrix} = \begin{pmatrix}2\\1\end{pmatrix}$
}

We know what a basis is and what know what orthogonal vectors are.  If you think carefully
about it, all of your mathematical life you have been dealing with coordinate systems that have
orthogonal bases.  Every time you graph in $\mathbb{R}^2$ or $\mathbb{R}^3$ you've used
the idea of othogonality without thinking much about it.  Let's see why this is so
incredibly useful.

\begin{problem}
    If $\mathcal{B} = \{\bv_1, \bv_2, \ldots, \bv_n\}$ is an orthogonal basis for a vector
    space $\mathcal{V}$ then how do you write $\bx$ as a linear combination of the basis vectors?\\
    (Why is it advantageous to have an orthogonal basis?) 
    Hint: Since $\bx = c_1 \bv_1 + c_2 \bv_2 + \cdots + c_n \bv_n$ how can you use
    orthogonality to solve for $c_j$?
\end{problem}
\solution{
    $c_j = \frac{\bx \cdot \bv_j}{\bv_j \cdot \bv_j}$
}

\begin{problem}
    Implement your idea on the subspace spanned by the basis
    \[ \mathcal{B} = \left\{ \begin{pmatrix} 1\\4\\-3 \end{pmatrix}\, , \, \begin{pmatrix}
            3 \\ 0 \\ 1 \end{pmatrix} \right\} \]
    where $\bx$ is in the subspace of $\mathbb{R}^3$ spanned by $\mathcal{B}$.
    Specifically, let $\bx$ be defined as
    \[ \bx = \begin{pmatrix} 2 \\ -4\\ 1 \end{pmatrix} \]
\end{problem}
\solution{
    Since $\bx = c_1 \begin{pmatrix} 1 \\ 4 \\ -3 \end{pmatrix} + c_2 \begin{pmatrix} 3 \\ 0 \\
        1 \end{pmatrix}$ so 
    \[ c_1 = \frac{\bx \cdot \bv_1}{\bv_1 \cdot \bv_1} = \frac{-17}{26} \]
    \[ c_2 = \frac{\bx \cdot \bv_2}{\bv_2 \cdot \bv_2} = \frac{7}{10} \]
}

Now summarize the process that you built in the previous problem into the following
theorem.
\begin{thm}[Building Vectors from an Orthogonal Basis]\label{thm:orthogonal_basis}
    If $\mathcal{B} = \{\bv_1, \bv_2, \ldots, \bv_n\}$ is an orthogonal basis for a vector
    space $\mathcal{V}$ then for any vector $\bx \in \mathcal{V}$ we can write
    \[ \bx = C_1 \bv_1 + C_2 \bv_2 + \cdots + C_n \bv_n \]
    where
    \[ C_k = \underline{\hspace{1in}}. \]
%     \[ \bx = \underline{\hspace{0.25in}} \bv_1 + \underline{\hspace{0.25in}} \bv_2 +
%     \underline{\hspace{0.25in}} \bv_3 + \cdots + \underline{\hspace{0.25in}} \bv_n \]
%     (Fill in the blanks)
\end{thm}
\begin{proof}
    (prove this theorem by leveraging the fact that we have an orthogonal basis)
\end{proof}
\solution{
    $c_j = \frac{\bx \cdot \bv_j}{\bv_j \cdot \bv_j}$
}


\begin{thm}
    If the nonzero vectors $\bu_1, \bu_2, \bu_3, \dots, \bu_k$ are mutually orthogonal
    then they are linearly independent.
\end{thm}
\begin{proof}
    (prove this theorem)
\end{proof}
\solution{
    Consider $\bo = \sum_{j=1}^n c_j \bu_j$.  From the previous theorem we know that $c_j
    = (\bo \cdot \bu_j) / (\bu_j \cdot \bu_j) = 0$.  Therefore the only solution is the
    trivial solution and the vectors must be linearly independent.
}


\begin{problem}
    Determine if the following set of vectors is linearly independent.  Do this two
    different ways.
    \[ \left\{ \begin{pmatrix} 3 \\ 1 \\ 1 \end{pmatrix} \, , \, \begin{pmatrix} -1 \\ 2
            \\ 1
    \end{pmatrix} \, , \, \begin{pmatrix} -1 \\ -4 \\ 7 \end{pmatrix} \right\} \]
\end{problem}
\solution{
They are mutually orthogonal so they are linearly independent.
}







\begin{problem}
    If we have two linearly independent vectors that are NOT orthogonal, how do
    we find a set of two orthogonal vectors that span the same space?

    For example, can we find two orthogonal vectors that span the same space as
    \[ \begin{pmatrix} 2 \\ 0 \end{pmatrix} \quad \text{and} \quad \begin{pmatrix} 1
        \\ 1 \end{pmatrix} \]
\end{problem}

\subsection{The Gram-Schmidt Process: Making Orthogonal Sets}
The previous theorems and problems give us good reason to think that having an orthogonal
(or orthonormal) basis for a vector space is advantageous both computationally and
geometrically.  In fact, we have been used to an orthonormal basis all of our matheamtical
lives since that is what the regular Cartesian coordinate system is built from.  The
question now is this: \\
Given a basis $\mathcal{B}$ for a vector space $\mathcal{V}$ how can we transform that
basis into a different basis for the same space but also gain orthogonality?
We will build your intuition to the process via a scaffolded problem.  

\begin{problem}
    Build a basis for $\mathbb{R}^2$ so that it contains two orthogonal unit vectors with
    one of the vectors parallel to $\bv_1 = \begin{pmatrix} 1\\1\end{pmatrix}$.
\end{problem}

\begin{problem}
    Consider the vector space $\mathbb{R}^3$ with the basis $\mathcal{B} = \{\bv_1, \bv_2,
    \bv_3\}$ given by
    \[ \bv_1 = \begin{pmatrix} 1\\1\\1\end{pmatrix}, \quad \bv_2 = \begin{pmatrix} 0
            \\1\\1\end{pmatrix},\quad \bv_3 = \begin{pmatrix} 0\\0\\1\end{pmatrix} \]
                We are going to build a basis $\mathcal{U} = \{\bu_1, \bu_2, \bu_3\}$ such that $\text{span}(\mathcal{U}) =
    \mathbb{R}^3$ but the vectors are also mutually orthogonal and all have unit length.
    (One should note here that the normalization step is optional but since unit vectors
    are so nice to work with we are leaving it here.)
    \begin{enumerate}
        \item[(a)] Define $\bu_1$ as a unit vector that points in the same direction as
            $\bv_1$.
            \[ \bu_1 = \frac{1}{\|\bv_1\|} \bv_1 =  \begin{pmatrix} \underline{\hspace{0.25in}} \\
                    \underline{\hspace{0.25in}} \\ \underline{\hspace{0.25in}} \end{pmatrix}
                    \]
\solution{
    $\bu_1 = \frac{1}{\sqrt{3}} \begin{pmatrix} 1\\1\\1\end{pmatrix} = \begin{pmatrix}
        1/\sqrt{3} \\ 1/\sqrt{3} \\ 1/\sqrt{3} \end{pmatrix}$
}
        \item[(b)] Now we project $\bv_2$ onto $\bu_1$ and find the error in the
            projection.  This would be the vector $\bw$ in Figure \ref{fig:proj_R2}.  Once
            we have the error we should normalize it to get $\bu_2$.
            \[ \bw_2 = \bv_2 - \text{proj}_{\bu}(\bv_2) = \bv_2 - \left( \bv_2 \cdot \bu_1
                \right) \bu_1 \quad \text{and therefore} \quad \bu_2 =
                \frac{1}{\|\bw_2\|}\bw_2. \]
            Draw a picture of what we just did.
            \[ \bu_2 = \frac{1}{\|\bv_1\|} \bv_1 =  \begin{pmatrix}
                    \underline{\hspace{0.25in}} \\ \underline{\hspace{0.25in}} \\
                    \underline{\hspace{0.25in}} \end{pmatrix} \]
\solution{
    $\bw_2 = \begin{pmatrix} 0\\1\\1\end{pmatrix} - \left( \frac{2}{\sqrt{3}}
        \right)\begin{pmatrix} 1/\sqrt{3} \\ 1/\sqrt{3} \\ 1/\sqrt{3} \end{pmatrix} =
            \begin{pmatrix}0\\1\\1\end{pmatrix} - \begin{pmatrix} 2/3 \\ 2/3 \\ 2/3
            \end{pmatrix} = \begin{pmatrix} -2/3 \\ 1/3 \\ 1/3 \end{pmatrix}$.  Therefore
                $\bu_2 = \frac{1}{\|\bw_2\|}\bw_2 $ so
                \[ \bu_2 = \frac{\sqrt{3}}{\sqrt{2}} \begin{pmatrix} -2/3 \\ 1/3 \\
                        1/3\end{pmatrix} = \frac{1}{\sqrt{6}} \begin{pmatrix}
                            -2\\1\\1\end{pmatrix} = \begin{pmatrix} -2/\sqrt{6} \\
                                1/\sqrt{6} \\ 1/\sqrt{6} \end{pmatrix} \]
}
        \item[(c)] For $\bu_3$ we project $\bv_3$ onto both $\bu_1$ and $\bu_2$ and then
            normalize.
            \[ \bw_3 = \bv_3 - \left( \bv_3 \cdot \bu_1 \right)\bu_1 - \left( \bv_3 \cdot
                \bu_2
                \right) \bu_2 \quad \text{and therefore} \quad \bu_3 =
                \frac{1}{\|\bw_3\|}\bw_3 \]
            \[ \bu_3 = \frac{1}{\|\bv_1\|} \bv_1 =  \begin{pmatrix}
                    \underline{\hspace{0.25in}} \\ \underline{\hspace{0.25in}} \\
                    \underline{\hspace{0.25in}} \end{pmatrix} \]
\solution{
    \[ \bw_3 = \bv_3 - \left( \frac{1}{\sqrt{3}} \right)\begin{pmatrix} 1/\sqrt{3} \\
            1/\sqrt{3} \\ 1/\sqrt{3} \end{pmatrix} - \left( \frac{1}{\sqrt{6}} \right)
            \begin{pmatrix} -2/\sqrt{6} \\ 1/\sqrt{6} \\ 1/\sqrt{6} \end{pmatrix} =
                \begin{pmatrix}0\\0\\1\end{pmatrix} - \begin{pmatrix}1/3 \\ 1/3 \\ 1/3
                \end{pmatrix} - \begin{pmatrix} -2/6 \\ 1/6 \\ 1/6 \end{pmatrix} =
                    \begin{pmatrix} 0 \\ -1/2 \\ 1/2 \end{pmatrix} =
                        \frac{1}{2} \begin{pmatrix}0\\-1\\1\end{pmatrix} \]
    Therefore, we can build $\bu_3$ by normalizing
    \[ \bu_3 = \frac{\sqrt{2}}{2} \begin{pmatrix} 0\\-1\\1 \end{pmatrix}  =
            \frac{1}{\sqrt{2}} \begin{pmatrix} 0\\-1\\1\end{pmatrix} = \begin{pmatrix}
                0\\-1/\sqrt{2} \\ 1/\sqrt{2} \end{pmatrix}\]
}
        \item[(d)] Verify that indeed $\mathcal{U} = \{\bu_1, \bu_2, \bu_3\}$ is an
            orthonormal basis for $\mathbb{R}^3$.
\solution{
The orthonormal basis is:
\[ \mathcal{U} = \left\{ \begin{pmatrix} 1/\sqrt{3} \\ 1/\sqrt{3} \\ 1/\sqrt{3}
    \end{pmatrix} \, , \, \begin{pmatrix} -2/\sqrt{6} \\ 1/\sqrt{6} \\ 1/\sqrt{6}
    \end{pmatrix} \, , \, \begin{pmatrix} 0 \\ -1/\sqrt{2} \\ 1/\sqrt{2} \end{pmatrix}
    \right\} \]
    and we leave it to the reader to verify that indeed the vectors are mutually
    orthogonal.
}
    \end{enumerate}
\end{problem}


\begin{problem}
    Use the Gram-Schmidt process outlined in the previous problem to produce an orthogonal
    basis $\mathcal{U}$ for the subspace spanned by
    \[ \begin{pmatrix} 3\\0\\-1\end{pmatrix} \quad \text{and} \quad \begin{pmatrix}
            8\\5\\-6\end{pmatrix}. \]
\end{problem}

\begin{problem}
    Let's build an orthogonal basis $\mathcal{U}$ for $\mathbb{R}^3$.  To get started let
    $\bu_1 = \begin{pmatrix} 1\\0\\1\end{pmatrix}$.  (notice that we are not
        normalizing this time)
        \begin{enumerate}
            \item[(a)] Create a vector $\bu_2$ in $\mathbb{R}^3$ so that $\bu_1 \perp
                \bu_2$.
\solution{
A simple choice is $\bu_2 = (0,1,0)^T$.
}
            \item[(b)] Pick a vector $\bv_3 \in \mathbb{R}^3$ such that $\bv_3$ is
                linearly independent of $\bu_1$ and $\bu_2$.  Then use one step of the
                Gram-Schmidt process to create $\bu_3$ out of $\bv_3$.
\solution{One choice is $\bv_3 = (2,0,1)^T$.  Therefore
    \[ \bu_3 = \bv_3 - \left( \frac{\bv_3 \cdot \bu_1}{\|\bu_1\|^2} \right)\bu_1 - \left(
        \frac{\bv_3 \cdot \bu_2}{\|\bu_2\|^2}\right) \bu_2 = \begin{pmatrix}
            2\\0\\1\end{pmatrix} - \frac{3}{2}\begin{pmatrix} 1\\0\\1\end{pmatrix} - 0
                \begin{pmatrix} 0\\1\\0\end{pmatrix} = \begin{pmatrix} 1/2 \\ 0 \\
                    -1/2\end{pmatrix} \]
}
            \item[(c)] Verify that the vectors in your proposed basis are indeed mutually
                orthogonal.  If so we can use one of the previous theorems (which one) to
                say that the vectors are linearly independent and must therefore span
                $\mathbb{R}^3$.
\solution{It is trivial to verify that the three vectors are indeed mutually orthogonal.
}
        \end{enumerate}
\end{problem}


\newpage\section{Inner Product Spaces}
Now time for some more abstraction!  In this section we take the notions of geometry and
abstract them to generalized vector spaces.  You may have noticed that the dot product is
the basic computation necessary to understand angle in $\mathbb{R}^n$ so we first have to
provide a generalized version of the dot product.

\begin{definition}[The Inner Product]\label{def:inner_product}
    An {\bf inner product} is the abstraction of a dot product to a general vector space.
    If $\bu$, $\bv$, and $\bw$ are vectors in a vector space $\mathcal{V}$ and $c$ is some real
    number then
        \begin{enumerate}
            \item $\left< \bu,\bv\right> = \left<\bv,\bu\right>$
            \item $\left< \bu,\bv + \bw\right> = \left< \bu,\bv\right> + \left<
                \bu,\bw\right>$
            \item $\left< c\bu,\bv\right> = c \left< \bu,\bv\right>$
            \item $\left< \bu,\bu\right> \ge 0$ and $\left<\bu,\bu\right>=0$ if and only if
                $\bu=\bo$
        \end{enumerate}
\end{definition}

\begin{problem}
    Verify that the dot product is indeed an inner product on the vector space
    $\mathbb{R}^n$. 
\end{problem}
\solution{
    \begin{enumerate}
        \item The dot product is definitely symmetric: $\bu \cdot \bv = \bv \cdot \bu$.
            Proof:
            \[ \bu \cdot \bv = \sum_{j=1}^n u_j v_j = \sum_{j=1}^n v_j u_j = \bv \cdot \bu
            \]
    \item $\bu \cdot (\bv + \bw) = (\bu \cdot \bv) + (\bu + \bw)$ since
        \[ \bu \cdot (\bv + \bw) = \sum_{j=1}^n u_j(v_j+w_j) = \sum_{j=1}^n u_j v_j + u_j
            w_j = \sum_{j=1}^n u_j v_j + \sum_{j=1}^n u_j w_j = \bu \cdot \bv + \bu \cdot
        \bw \]
    \item $(c\bu) \cdot \bv = c (\bu \cdot \bv)$ since
        \[ (c\bu) \cdot \bv = \sum_{j=1}^n (cu_j)v_j = c \sum_{j=1}^n u_j v_j = c (\bu
        \cdot \bv) \]
    \item If $\bu = \bo$ then $\bu \cdot \bu = \sum_{j=1}^n u_j = \sum_{j=1}^n 0 = 0$.
        Furthermore, $\bu \cdot \bu = \sum_{j=1}^n u_j^2$ which is the sum on non-negative
        real numbers which must clearly also be non-negative.
    \end{enumerate}
}


\begin{problem}
    An inner product on $\mathcal{P}_2$ (the space of all quadratic polynomials) is 
    \[ \left< p \,,\, q \right> = p_0 q_0 + p_1 q_1 + p_2 q_2 \]
    where $p(x) = p_0 + p_1x + p_2 x^2$ and $q(x) = q_0 + q_1 x + q_2 x^2$.  Verify that
    this indeed is a proper inner product on $\mathcal{P}_2$.  If it is then find the
    inner product of $p(x) = x^2 + 1$ and $q(x) = 2x+x^2$ as well as the angle between
    $p(x)$ and $q(x)$.
\end{problem}


\begin{problem}
    Consider the vector space of quadratic polynomials on the interval $x \in [0,1]$.
    \[ \mathcal{P}_2 = \{a_0 + a_1 x + a_2 x^2 \, : \, a_0, a_1, a_2 \in
        \mathbb{R} \text{ and } x \in [0,1] \} \]
    An inner product on this vector space is
    \[ \left< f,g\right> =
    \int_0^1 f(x) g(x) dx. \] 
    \begin{enumerate}
        \item[(a)] Verify that this is indeed a proper inner product on $\mathcal{P}_2$
        \item[(b)] Find the inner product of $f(x) = x^2+1$ and $g(x) =
            2x+x^2$ in $\mathcal{P}_2$ under this inner product.
        \item[(c)] Set up the necessary integrals to find the lengths of $f$ and $g$ in
            $\mathcal{P}_2$ under this inner product.
        \item[(d)] Set up the necessary integrals to find the angle between $f$ and $g$ in
            $\mathcal{P}_2$ under this inner product.
        \item[(e)] Is this the only inner product on $\mathcal{P}_2$?
    \end{enumerate}

\end{problem}
\solution{
    \begin{enumerate}
        \item[(b)] $\left< f,g\right>= \int_0^1 (x^2+1)(2x+x^2) dx = \int_0^1 2x^3 + x^4 + 2x + x^2 dx = \frac{1}{2} +
            \frac{1}{5} + 1 + \frac{1}{3} = \frac{61}{30}$
        \item[(c)] $\|f\| = \left< f,f\right>^{1/2} = \sqrt{\int_0^1 (x^2+1)^2 dx}$, and
            $\|g\| = \left<g,g\right>^{1/2} = \sqrt{\int_0^1 (2x+x^2)^2 dx}$.
        \item[(d)] $\theta = \frac{\left< f,g\right>}{\|f\|\|g\|}$
        \item[(e)] since these are polynomials we could take any finite bounds of
            integration and we get a valid inner product.
    \end{enumerate}
}


\begin{problem}
    Consider the vector space of $2 \times 2$ real matrices 
    \[ \mathcal{V} = \left\{ \begin{pmatrix} a & b \\ c & d \end{pmatrix} \, : \,
    a,b,c,d \in \mathbb{R} \right\} \]
    along with the inner product $\left<A \,,\, B \right> = \text{trace}(AB^T)$.  
    Note: If $M$ is a matrix, the {\it trace} of the matrix, $tr(M)$, is the sum
    of the entries on the main diagonal.

    To
    simplify your computations a bit we'll expand the definition of the inner product.
    \[ \left<A\,,\,B\right> = \text{trace}(AB^T) = \text{trace}\left( 
        \begin{pmatrix} a_{11} & a_{12} \\ 
                        a_{21} & a_{22} \end{pmatrix} 
        \begin{pmatrix} b_{11} & b_{21} \\
        b_{12} & b_{22} \end{pmatrix} \right) = \text{trace}\left( 
        \begin{pmatrix} a_{11} & a_{12} \\ 
                        a_{21} & a_{22} \end{pmatrix} 
        \begin{pmatrix} b_{11} & b_{12} \\
        b_{21} & b_{22} \end{pmatrix}^T \right)\]
    \[ = \text{trace}\left(
                    \begin{pmatrix} a_{11}b_{11} + a_{12} b_{12} & a_{11}b_{21} +
                        a_{12}b_{22} \\ a_{21}b_{11} + a_{22}b_{12} & a_{21} b_{21} +
                    a_{22} b_{22} \end{pmatrix} \right) \]
    \[ = \left( a_{11}b_{11} + a_{12} b_{12} \right) + \left(  a_{21} b_{21} + a_{22}
    b_{22}\right) \]
    \begin{enumerate}
        \item[(a)] Why is this a natural choice for the inner product between $2 \times 2$ matrices?
        \item[(b)] Find an orthogonal basis for $\mathcal{V}$.  That's right \dots I'm asking you to find
            angles between matrices!! AWESOME!!
    \end{enumerate}
\end{problem}
\solution{
    This is a natural choice since it is essentially equivalent to the dot product in
    $\mathbb{R}^4$.
    \[ \begin{pmatrix} 1 & 0 \\ 0 & 0 \end{pmatrix}, 
        \begin{pmatrix} 0 & 1 \\ 0 & 0 \end{pmatrix},
        \begin{pmatrix} 0 & 0 \\ 1 & 0 \end{pmatrix},
    \begin{pmatrix} 0 & 0 \\ 0 & 1 \end{pmatrix} \]
}




% \begin{problem}[Geometry on Periodic Functions]
%     Consider the vector space 
%     \[ \mathcal{V} = \{ f(x) \, : \, f(x) \text{ is } 2\pi \text{ periodic }\} \]
%     along with the inner product 
%     \[ \left<f,g\right> = \int_0^{2\pi} f(x) g(x) dx \]  
%     Suggest a basis for $\mathcal{V}$ and determine if it is an orthogonal basis.
% \end{problem}
% \solution{
%     \[ \mathcal{B} = \{\sin(kx),\cos(kx)\}_{k=0}^\infty \]
%     \[ \int_0^{2\pi} \sin(kx) \sin(jx) dx = \left\{ \begin{array}{ll} \pi &
%         j=k \\ 0 & j \ne k \end{array} \right. \]
%     \[ \int_0^{2\pi} \sin(kx) \cos(jx) dx = 0 \quad \forall j,k \]
%     \[ \int_0^{2\pi} \cos(kx) \sin(jx) dx = \left\{ \begin{array}{ll} \pi &
%         j=k \\ 0 & j \ne k \end{array} \right. \]
% }

\newpage\section{Fourier Series}
In this section we will consider one of the most beautiful and useful
applications of inner product spaces.  This application has, quite literally, changed the modern world in uncountably
many ways.  This application, which will arise later in these notes (in the PDE's
chapter), is one of the most stunningly beautiful applications out there for everyone to
see: The Fourier Series.  

Fourier series were first developed by Joseph Fourier (ca 1800) as part of the development
of the theory of heat transport.  Since then the field of Fourier Analysis has taken over
physics, engineering, signal processing, computer science, \ldots and the list goes on and
on and on. Every signal that we transmit is touched in some way by the ideas developed by
Fourier in the 1800's.  It is no exaggeration that every person in our technological world is impacted
daily by Fourier Analysis. 

Let's get into it.  Consider the vector space spanned by an infinite basis of sine functions 
\[ \mathcal{B} = \left\{ \sin\left( k x\right) \, : \, k \in \mathbb{N} \right\} \]
equipped with the inner product 
\begin{flalign}
    \left< f , g \right> = \frac{1}{\pi} \int_0^{2\pi} f(x) g(x) dx.
    \label{eqn:fourier_ip}
\end{flalign}
This particular basis is infinite dimensional since the natural numbers, $\mathbb{N}$, are
(countably) infinite, and if we consider the span we can build any periodic function
$f(x)$ where $f(0) = 0$ as a
linear combination of sine functions of different frequencies.
More specifically, since every periodic function can be written as a linear combination of the
basis functions we have the infinite sum
\begin{flalign}
    f(x) = \sum_{k=1}^\infty C_k \sin\left( k x\right) \label{eqn:fourier_sine}
\end{flalign}
for all period functions $f$. The most important part of the basis $\mathcal{B}$ is that
it is an othonormal basis under the inner product: an orthogonal basis made entirely of
unit vectors.  
\begin{thm}\label{thm:fourier_sine_ip}
    If $j,k \in \mathbb{N}$ then
    \begin{flalign}
        \left< \sin(kx) \,,\, \sin(jx) \right> = \frac{1}{\pi} \int_0^{2\pi} \sin(kx)
        \sin(jx) dx = \left\{ \begin{array}{ll} 0, & j \ne k \\ 1, & j=k \end{array}
        \right.
        \label{eqn:fourier_ip_1}
    \end{flalign}
\end{thm}
\begin{problem}
    What does Theorem \ref{thm:fourier_sine_ip} say about the functions $\sin(kx)$ and
    $\sin(jx)$ under the inner product
    \[ \left< f(x) \, , \, g(x) \right> = \frac{1}{\pi} \int_0^{2\pi} f(x) g(x) dx? \]
\end{problem}

\begin{proof}
    Let's prove Theorem \ref{thm:fourier_sine_ip}.  Notice that in Theorem
    \ref{thm:fourier_sine_ip} if we let $k=j$ then the inner product shows that 
    \[ \| \sin(kx) \| = \sqrt{ \left< \sin(kx) \, , \, \sin(kx) \right>} =
    \sqrt{\frac{1}{\pi} \int_0^{2\pi} \sin^2(kx) dx } = 1 \]
    which implies that each of the sine functions is a unit vector in this inner product
    space.  Furthermore, if $j \ne k$ then $\left< \sin(kx) \,,\, \sin(jx)\right> = 0$,
    which shows that $\sin(kx)$ and $\sin(jx)$ are orthogonal in this space.
    
    Let's integrate directly to verify this.  First, if recall the trig identity
    \[ 2\sin(\theta)\sin(\phi) = \cos(\theta - \phi) - \cos(\theta + \phi) \]
    we can rewrite \eqref{eqn:fourier_ip_1} as
    \[ \left< \sin(kx) , \sin(jx) \right> = \frac{1}{\pi} \left[ \frac{1}{2} \int_0^{2\pi}
    \cos\left( (k-j)x \right) - \cos\left( (k+j)x  \right) dx  \right]. \]

    There are clearly two cases: when $k=j$ and when $k\ne j$. 
    In the case that $k=j$ we observe that $k-j=0$ so the integral becomes 
    \[ \frac{1}{2\pi} \int_0^{2\pi} 1 - \cos(2kx) dx \]
    which we can integrate to 
    \[ \frac{1}{2\pi} \int_0^{2\pi} 1-\cos(2kx) dx = \frac{1}{2\pi} \left[ x -
        \frac{1}{2k}\sin(2kx)
    \right]_0^{2\pi} = \frac{1}{2\pi} \left[ 2\pi - \frac{\sin(4k\pi)}{2k} +
    \frac{\sin(0)}{2k} \right] = 1 \]
    since the sine of integer multiples of $\pi$ is zero.  

    In the case that $k \ne j$ we have
    \[ \left< \sin(kx) \, , \, \sin(jx) \right> = \frac{1}{2\pi} \left[ \int_0^{2\pi} \cos\left(
    (k-j)x \right) dx - \int_0^{2\pi} \cos\left( (k+j)x \right) dx \right] \]
    which can be integrated to
    \[ \frac{1}{2\pi} \left[ \frac{\sin\left( (k-j)x \right)}{k-j} - \frac{\sin\left(
    (k+j)x \right)}{k+j} \right]_0^{2\pi} = \frac{1}{2\pi} \left[ \frac{\sin\left(2\pi (k-j) \right)}{k-j} - \frac{\sin\left(
    2\pi(k+j) \right)}{k+j} \right] = 0 \] 
    where we again have used the fact that the sine of an integer multiple of $\pi$ is
    zero along with the fact that $\sin(0) = 0$.

    This concludes the proof and we see that the set $\mathcal{B} = \left\{ \sin(kx) \, : \, k
        \in \mathbb{N} \right\}$ is indeed an orthonormal basis of functions under
        inner product \eqref{eqn:fourier_ip}.
\end{proof}


If you aren't happy with the analytical proof you can at least {\it convince} yourself
that indeed Theorem \ref{thm:fourier_sine_ip} is true.
\begin{problem}
    Open MATLAB (or any other symbolic calculus package) and verify that the basis
\[ \mathcal{B} = \left\{ \sin\left( k x\right) \, : \, k \in \mathbb{N} \right\} \]
    is indeed an orthogonal basis.  That is, compute 
    \[ \left< \sin\left( k x\right) \, , \, \sin\left( j x \right)
        \right> =  \frac{1}{\pi} \int_0^{2\pi} \sin\left( k x\right) \sin\left( j x\right) dx \]
    for various values of $j$ and $k$ and verify that 
    \begin{itemize}
        \item if $j=k$ then the inner produce is identically 1.
        \item if $j \neq k$ then the inner product is zero.
    \end{itemize}
\end{problem}

It is worth it to note that we could have defined the inner product for this space as an
integral from $-\pi$ to $\pi$ instead of $0$ to $2\pi$ and all of these results would
still hold.  After all, the functions are periodic with period $2\pi$ so any domain with
length $2\pi$ would work just fine.

\begin{thm}
    Any periodic function $f(x)$ with period $2\pi$ and $f(0) = 0$ can be written as a
    linear combination of sine functions $\sin(kx)$.  That is,
    \[ f(x) = C_1 \sin(1x) + C_2 \sin(2x) + C_3 \sin(3x) + \cdots = \sum_{k=1}^\infty C_k
    \sin(kx). \]
\end{thm}
\begin{proof}
    Indeed, we have just expanded the periodic function as a linear combination of the
    basis functions.
\end{proof}

\begin{problem}
    Go to
    \href{http://mathlets.org/mathlets/fourier-coefficients/}{http://mathlets.org/mathlets/fourier-coefficients/},
    choose ``Target A'', ``Sine'', and ``All terms''.  Then use the sliders on the right
    to closely match the square wave with a Fourier sine series.
\end{problem}

\begin{problem}
    If $f(x)$ is some periodic function then we can write it as a linear combination of
    the basis vectors in $\mathcal{B}$:
    \[ f(x) = \sum_{k=1}^{\infty} C_k \sin(k x). \]
    Knowing that the sine functions in the sum form an orthonormal basis for the space of
    periodic functions propose a way to find each $C_k$.  Hint: Consider Theorem
    \ref{thm:orthogonal_basis}.
\end{problem}


\begin{thm}[Fourier Sine Series Coefficients]
    If $f(x)$ is a $2\pi$ periodic function with $f(0) =0$ then we can expand $f(x)$ as
    the series 
    \[ f(x) = \sum_{k=1}^\infty C_k \sin(kx) \]
    where 
    \[ C_k = \left< f(x) \, , \, \sin(kx) \right> = \frac{1}{\pi} \int_0^{2\pi} f(x)
    \sin(kx) dx. \]
\end{thm}
\begin{proof}
    This theorem is a special case of Theorem \ref{thm:orthogonal_basis}.
\end{proof}

\begin{center}
    {\bf Now for some fun!}
\end{center}
\begin{problem}
    Let's build a sound signal with some noise and then use the mathematics that we just
    discussed to work some magic. 
    \begin{enumerate}
        \item[(a)] We'll start by building a clean signal.

\begin{lstlisting}
clear; clc;
dt = 0.001; % set up a time step
t = 0:dt:2*pi % set up time from 0 to 2pi

% Now we'll buid a signal (like a dial tone)
% out of several sine waves with different amplitudes and 
% different frequencies (make up your own)
signal = 2*sin(240*t) + 5*sin(380*t) + 2.5*sin(700*t); 
soundsc(signal) % play the sound (turn up the volume a bit)
\end{lstlisting}

        \item[(b)] Now let's add some noise to the signal

\begin{lstlisting}
clear; clc;
dt = 0.001; % set up a time step
t = 0:dt:2*pi % set up time from 0 to 2pi

NoiseLevel = 10;
signal = 2*sin(240*t) + 5*sin(380*t) + 2.5*sin(700*t) + ... 
            NoiseLevel*rand(size(t)) - NoiseLevel/2; 
soundsc(signal) % play the sound (turn up the volume a bit)
\end{lstlisting}

\item[(c)] Now we'll pull the dominant frequencies out of the signal.  We do this by
    taking advatnage of the orthogonal basis for the vector space of periodic functions.
    Read the code below carefully and be sure you know exactly what it happening. \\
    Note: the \mcode{trapz} function does the trapezoidal rule to approximate the integral
    (yeah MATLAB!!).
\begin{lstlisting}
frequences = 1:1000; % list the frequencies we will try
for n=frequencies % try every frequency
    c(n) = (1/pi)*trapz(signal .* sin(n*t))*dt; % integrate
end
\end{lstlisting}

\newpage

\item[(d)] Let's see which frequencies it found.
\begin{lstlisting}
DominantFrequencies = frequencies( c > 1 )
\end{lstlisting}

\item[(e)] Let's make a plot of the signal and the Fourier weights.
\begin{lstlisting}
subplot(1,3,1)
plot(t,signal) % plot noisy signal
xlabel('time'), ylabel('intensity'), title('Noisy Signal')
axis([0,0.1,-NoiseLevel , NoiseLevel])

subplot(1,3,2)
plot(frequences, c, 'r')
xlabel('frequency'), ylabel('intensity'), 
title('Fourier Transform of Signal')
\end{lstlisting}

\item[(f)] And finally for the coolest part!!  Let's trim out the noise!!  We know what
    the dominant frequencies are, so we can build a signal directly from them.  This is
    EXACTLY how digital signal processing works!!
\begin{lstlisting}
CleanSignal = zeros(size(t)); % set up space for the clean signal
for n=DominantFrequencies
    CleanSignal = CleanSignal + c(n) * sin(n*t);
end
pause(2) % let matlab finish playing the old sound
soundsc(CleanSignal)

subplot(1,3,3)
plot(t,CleanSignal)
axis([0,0.1,-NoiseLevel,NoiseLevel])

\end{lstlisting}

\item[(g)] Now go have some fun!  Some explorations to consider:
    \begin{itemize}
        \item How much noise can you add and still reasonably recover the original signal?
        \item How complex can you make the signal and still recover it?
        \item Can you (audibly) hide a single note but recover it perfectly?
        \item \ldots
    \end{itemize}

\end{enumerate}

\end{problem}


\newpage

\begin{problem}
    Consider the function 
    \[ f(x) = \left\{ \begin{array}{cl} 1 & \text{ if } 0 < x < \pi \\ -1 & \text{ if }
            \pi < x < 2\pi \end{array} \right..\]
    We'll define $f(x)$ to have the value 0 at $0, \pm \pi, \pm 2\pi, \cdots$ and extend
    the function periodically with period $2\pi$ forever.  This way $f(x)$ is defined on
    all real numbers. 
    We want to build a Fourier Series for this function (called the {\it square wave})
    \[ f(x) = \sum_{k=1}^\infty C_k \sin(kx). \]
    \begin{enumerate}
        \item[(a)] From the previous problem you should have found that taking the inner
            product of $f(x)$ and $\sin(kx)$ for every value of $k$ will result in the
            value of $C_k$.  Use this idea to find $C_1$. 
            \[ C_1 = \left< f(x) \, , \, \sin(1x) \right> = \frac{1}{\pi} \int_0^{2\pi}
            f(x) \sin(1x) dx = \underline{\hspace{0.5in}} \]
            (Hint: The integral can be broken into two relatively easy integrals if you think carefully about
            $f(x)$.)
        \item[(b)] Now find a general formula for $C_k$ by examining the inner product
            \[ C_k = \left< f(x) \, , \, \sin(kx) \right> = \frac{1}{\pi} \int_0^{2\pi} f(x)
            \sin(kx) dx = \underline{\hspace{0.5in}} \]
        \item[(c)] Write the first several terms of the Fourier sine series for the square
            wave.
        \item[(d)] Use MATLAB to build a plot showing successive approximations of the
            square wave.
    \end{enumerate}
\end{problem}

\begin{problem}
    Repeat the previous problem to find the Fourier series for the function 
    \[ f(x) = -\frac{1}{\pi}x + 1 \]
    for $x \in [0,2\pi]$ and extended periodically outside the domain.
\end{problem}


% The notion of
% expanding periodic functions in terms of an infinite series of sines and cosines
% subsequently changed the world.  He gave birth to the mathematical field of harmonic
% analysis and is still used today in digital and analog signal processing.  
We will return to the idea of Fourier series in Chapter \ref{ch:PDEs} where we'll use Fourier series to
solve problems in heat conduction. We have also presented a bit of a
limited view in this section in that we have only allowed for Fourier {\it sine} series.
We can generalize the idea to a general Fourier series,
\[ f(x) = a_0 + \sum_{k=1}^\infty a_k \cos(kx) + \sum_{k=0}^\infty b_k \sin(kx), \]
where using both sine and cosine functions allows for more flexibility in the types of
functions we can model.  The choice of inner product doesn't change and the cosine terms
have all of the wonderful properties as we had before with the sine functions.  

% For now it seems as if it is an
% exercise in inner products and may even still seem a bit esoteric.  Let's do one more
% problem to illustrate how cool the Fourier series really is!
% 
% \begin{problem}
%     
% \end{problem}
% 
% 
%     Open MATLAB and complete the following partial code to plot an approximation to the
%     Fourier series of the square wave.
% \begin{lstlisting}
% clear; clc;
% syms x
% f(x) = 0*x; % this gives a placeholder for the function
% N = 20; % the top end of the finite Fourier series
% for k=1:N
%   C(k) = ... some code to find C(k) ...
%   f(x) = f(x) + C(k) * sin(k*x);
% end
% ezplot(f(x) , [0,2*pi])
% \end{lstlisting}
% Once you have the plot working, append the code
% \begin{lstlisting}
% x = 0:0.01:150;
% MySound = double(f(x));
% soundsc(MySound)
% \end{lstlisting}
% \ldots and turn the volume up.
% \end{problem}
% 


\newpage\section{Linear Transformations}

\begin{problem}
    Loosly speaking, a {\bf linear transformation} in $\mathbb{R}^n$ transforms one set of
    basis vectors to another.  Go to
    \href{https://shad.io/MatVis/}{https://shad.io/MatVis/} and use the slider bar to
    experiment with how the standard basis in $\mathbb{R}^2$ transitions to other bases.
    Specifically experiment with:
    \begin{itemize}
        \item[(a)] Can you create a linear transformation that only scales the coordinate
            system?
        \item[(b)] Can you create a linear transformation that only rotates the coordinate
            system?
        \item[(c)] What appears to happen to the determinant of the $1 \times 1$ square
            under your transformation?  What does this tell you about how the
            transformation changes areas in $\mathbb{R}^2$?
    \end{itemize}
\end{problem}

\begin{definition}[Linear Transformation]
    A {\bf linear transformation} $T$ from a vector space $\mathcal{V}$ into a vector
    space $\mathcal{W}$ is a rule that assigns to each vector $\bv \in \mathcal{V}$ a
    unique vector $\bw \in \mathcal{W}$, such that
    \begin{enumerate}
        \item[(a)] $T(\bv_1 + \bv_2) = T(\bv_1) + T(\bv_2)$ for all $\bv_1, \bv_2 \in \mathcal{V}$
        \item[(b)] $T(c\bv) = c T(\bv)$ for all $\bv \in \mathcal{V}$ and all scalars
            $c$.
    \end{enumerate}
    More simply, a linear transformation has the property that
    \[ T(c_1 \bv_1 + c_2 \bv_2) = c_1 T(\bv_1) + c_2 T(\bv_2) \]
    for all $\bv_1 ,\bv_2 \in \mathcal{V}$ and for all scalars $c_1$ and $c_2$.
\end{definition}


\subsection{Matrix Transformation}
\begin{problem}
    Verify that if $A$ is an $n \times n$ matrix then the function $T$ defined as $T(\bx) =
    A\bx$ is indeed a linear transformation.
\end{problem}
\solution{
Yep.  Matrix multiplication is a linear operation.
}

Since matrix multiplication is a linear transformation let's look at some of the common
matrix transformations.  Matrix transformations commonly used in computer graphics are
{\it dilation}, {\it shear}, and {\it rotation}.  Notice that the common geometric transformation of
{\it translation} is not a linear transformation since if we translate then the origin does not
stay fixed.
\begin{problem}
    Consider the square defined by the set of points $S =
    \{(0,0)\,,\,(1,0)\,,\,(1,1)\,,\,(0,1)\}$. Let $T(\bx)$ be a linear transformation
    defined as $T(\bx) = A\bx$ where $A$ is a matrix and $\bx \in S$.  Applying the
    transformation $T$ to the points in $S$ gives a new geometric shape.  Describe the
    geometric action of each of the following transformations by applying them to the
    points in $S$.
    \begin{enumerate}
        \item[(a)] $T(\bx) = A\bx$ where
            \[ A = \begin{pmatrix} 1 & 3 \\ 0 & 1 \end{pmatrix} \]
        \item[(b)] $T(\bx) = A\bx$ where
            \[ A = \begin{pmatrix} 1 & 0 \\ 3 & 1 \end{pmatrix} \]
        \item[(c)] $T(\bx) = A\bx$ where
            \[ A = \begin{pmatrix} 5 & 0 \\ 0 & 5 \end{pmatrix} \]
        \item[(d)] $T(\bx) = A\bx$ where
            \[ A = \begin{pmatrix} 0.2 & 0 \\ 0 & 0.2 \end{pmatrix} \]
        \item[(e)] $T(\bx) = A\bx$ where
            \[ A = \begin{pmatrix} 0 & -1 \\ 1 & 0 \end{pmatrix} \]
        \item[(f)] $T(\bx) = A\bx$ where
            \[ A = \begin{pmatrix} 0 & 1 \\ -1 & 0 \end{pmatrix} \]
    \end{enumerate}
\end{problem}
\solution{
shear parallel to the $x$, shear parallel to the $y$, dilation, dilation, rotation
counterclockwise, rotation clockwise
}

% \begin{problem}
%     Let $T$ be a linear transformation defined as $T(\bx) = A \bx$ for some $2 \times 2$
%     matrix $A$.  If 
%     \[ A \begin{pmatrix} 1 \\ 0 \end{pmatrix} = \begin{pmatrix} 1 \\ 0\end{pmatrix} \quad
%         \text{and} \quad A \begin{pmatrix} 0 \\ 1 \end{pmatrix} = \begin{pmatrix} 3 \\ 1
%     \end{pmatrix} \]
%     then what is the matrix $A$ and how would you describe the geometric transformation?
% \end{problem}
% \solution{
%     this is the same shear matrix as in part (a) from the previous problem
% }
% 
% \begin{problem}
%     Let $T$ be a linear transformation defined as $T(\bx) = A \bx$ for some $2 \times 2$
%     matrix $A$.  If 
%     \[ A \begin{pmatrix} 1 \\ 0 \end{pmatrix} = \begin{pmatrix} 5 \\ 0\end{pmatrix} \quad
%         \text{and} \quad A \begin{pmatrix} 0 \\ 1 \end{pmatrix} = \begin{pmatrix} 0 \\ 5
%     \end{pmatrix} \]
%     then what is the matrix $A$ and how would you describe the geometric transformation?
% \end{problem}
% \solution{
%     this is the same dilation matrix as in part (b) from the previous problem
% }
% 
% 
% \begin{problem}
%     Let $T$ be a linear transformation defined as $T(\bx) = A \bx$ for some $2 \times 2$
%     matrix $A$.  If 
%     \[ A \begin{pmatrix} 1 \\ 0 \end{pmatrix} = \begin{pmatrix} 0 \\ 1\end{pmatrix} \quad
%         \text{and} \quad A \begin{pmatrix} 0 \\ 1 \end{pmatrix} = \begin{pmatrix} -1\\ 0
%     \end{pmatrix} \]
%     then what is the matrix $A$ and how would you describe the geometric transformation?
% \end{problem}
% \solution{
%     this is the same rotation matrix as in part (c) from the previous problem
% }

\begin{definition}[Shear Matrices]
    The 2D linear transformation that geometrically shears parallel to the $x$ axis is
    defined as
    \[ T(\bx) = \begin{pmatrix} 1 & \underline{\hspace{0.25in}} \\
        \underline{\hspace{0.25in}} & 1 \end{pmatrix} \bx \]
    The 2D linear transformation that geometrically shears parallel to the $y$ axis is
    defined as
    \[ T(\bx) = \begin{pmatrix} 1 & \underline{\hspace{0.25in}} \\
        \underline{\hspace{0.25in}} & 1 \end{pmatrix} \bx \]
    (fill in the blanks)
\end{definition}

\begin{problem}
In Figure \ref{fig:lt_shear} we applied two different shear transformations to the set $S
= \{(1,0),(0,1),(-1,0),(0,-1)\}$.  Which shear transformations were applied?
\end{problem}
\solution{
    The shear parallel to the $x$ axis is achieved with $A = \begin{pmatrix} 1 & 2 \\ 0 &
        1 \end{pmatrix}$.  The shear parallel to the $y$ axis is achieved with $A =
    \begin{pmatrix} 1 & 0 \\ 2 & 1\end{pmatrix}$.
}

\begin{figure}[ht!]
    \begin{center}
        \begin{tikzpicture}
            \begin{axis}[axis equal, axis lines=center, grid, xmin=-2, xmax=2, ymin=-2, ymax=2,
                title={Shear Parallel to $x$ Axis}]
                \draw[thick, black] (axis cs:1,0) -- (axis cs:0,1) -- (axis cs:-1,0) --
                (axis cs:0,-1) -- (axis cs:1,0);
                \draw[thick, dashed, blue] (axis cs:1,0) -- (axis cs:2,1) -- (axis cs:-1,0) --
                (axis cs:-2,-1) -- (axis cs:1,0);
            \end{axis}
        \end{tikzpicture}
        \begin{tikzpicture}
            \begin{axis}[axis equal, axis lines=center, grid, xmin=-2, xmax=2, ymin=-2, ymax=2,
                title={Shear Parallel to $y$ Axis}]
                \draw[thick, black] (axis cs:1,0) -- (axis cs:0,1) -- (axis cs:-1,0) --
                (axis cs:0,-1) -- (axis cs:1,0);
                \draw[thick, dashed, red] (axis cs:1,2) -- (axis cs:0,1) -- (axis cs:-1,-2) --
                (axis cs:0,-1) -- (axis cs:1,2);
            \end{axis}
        \end{tikzpicture}
    \end{center}
    \caption{Examples of shear transformations. The blue transformation on the left is a
    shear parallel to the $x$ axis.  The red transformation on the right is a shear
parallel to the $y$ axis.}
    \label{fig:lt_shear}
\end{figure}

\begin{definition}[Dilation Matrices]
    The 2D linear transformation that geometrically dilates a figures by a factor of $r$ is defined as 
    \[ T(\bx) = \begin{pmatrix} \underline{\hspace{0.25in}} & 0 \\ 0 &
        \underline{\hspace{0.25in}} \end{pmatrix} \bx. \]
    (fill in the blanks)
\end{definition}

\begin{problem}
    In Figure \ref{fig:lt_dilation} we applied two different dilation transformations to the set $S =
    \{(0,0),(1,1),(1,2),(0,1)\}$.  Which dilation transformations were applied?
\end{problem}
\solution{
The left is a dilation by 2 and the right is a dilation by $-0.5$.
}

\begin{figure}[ht!]
    \begin{center}
        \begin{tikzpicture}
            \begin{axis}[axis equal, axis lines=center, grid, xmin=-1, xmax=4, ymin=-1,
                ymax=4, title={Dilation \#1}]
                \draw[thick, black] (axis cs:0,0) -- (axis cs:1,1) -- (axis cs:1,2) --
                (axis cs:0,1) -- (axis cs:0,0);
                \draw[thick, dashed, blue] (axis cs:0,0) -- (axis cs:2,2) -- (axis cs:2,4) --
                (axis cs:0,2) -- (axis cs:0,0);
            \end{axis}
        \end{tikzpicture}
        \begin{tikzpicture}
            \begin{axis}[axis equal, axis lines=center, grid, xmin=-2, xmax=2, ymin=-2, ymax=2,
                title={Dilation \#2}]
                \draw[thick, black] (axis cs:0,0) -- (axis cs:1,1) -- (axis cs:1,2) --
                (axis cs:0,1) -- (axis cs:0,0);
                \draw[thick, dashed, red] (axis cs:0,0) -- (axis cs:-0.5,-0.5) -- (axis
                cs:-0.5,-1) -- (axis cs:0,-0.5) -- (axis cs:0,0);
            \end{axis}
        \end{tikzpicture}
    \end{center}
    \caption{Examples of dilation transformations.}
    \label{fig:lt_dilation}
\end{figure}

\begin{definition}[Rotation Matrices]
    The 2D linear transformation that geometrically rotates a figure by an angle $\theta$
    is defined as 
    \[ T(\bx) = \begin{pmatrix} \cos(\theta) & -\sin(\theta) \\ \sin(\theta) &
        \cos(\theta) \end{pmatrix}. \]
\end{definition}

\begin{problem}
    What matrices yield to the following linear transformation?
    \begin{enumerate}
        \item[(a)] A rotation by $90^\circ$ counterclockwise.
        \item[(b)] A rotation by $90^\circ$ clockwise.
        \item[(c)] A rotation by $45^\circ$ counterclockwise.
        \item[(d)] A rotation by $30^\circ$ clockwise.
    \end{enumerate}
\end{problem}


\begin{figure}[ht!]
    \begin{center}
        \begin{tikzpicture}
            \begin{axis}[axis equal, axis lines=center, grid, xmin=-2, xmax=2, ymin=-2, ymax=2,
                title={Rotation by $30^\circ$ Counterclockwise}]
                \draw[thick, black] (axis cs:1,0) -- (axis cs:0,1) -- (axis cs:-1,0) --
                (axis cs:0,-1) -- (axis cs:1,0);
                \draw[thick, dashed, blue] (axis cs:0.866,0.5) -- (axis cs:-0.5,0.866) --
                (axis cs:-0.866,-0.5) -- (axis cs:0.5,-0.866) -- (axis cs:0.866,0.5);
            \end{axis}
        \end{tikzpicture}
        \begin{tikzpicture}
            \begin{axis}[axis equal, axis lines=center, grid, xmin=-3, xmax=2, ymin=-1, ymax=3,
                title={Rotation by $60^\circ$ Counterclockwise}]
                \draw[thick, black] (axis cs:0,0) -- (axis cs:1,1) -- (axis cs:1,2) --
                (axis cs:0,1) -- (axis cs:0,0);
                \draw[thick, dashed, red] (axis cs:0,0) -- (axis cs:-0.37,1.37)
                -- (axis cs:-1.23,1.87) -- (axis cs:-0.866,0.5) -- (axis
                cs:0,0);
            \end{axis}
        \end{tikzpicture}
    \end{center}
    \caption{Examples of rotation transformations. The blue transformation on the left is a
    rotation by $30^\circ$ counterclockwise.  The red transformation on the right is a
rotation by $60^\circ$ counterclockwise.}
    \label{fig:lt_rotation}
\end{figure}

Now that we know the back geometric linear transformations let's see what happens when we
compose several of them.  
\begin{problem}
    Consider the set of points $S = \{(0,0),(1,0),(1,1),(0,1)\}$.  \\(You may want to
    write code to complete this problem)
    \begin{enumerate}
        \item[(a)] Shear the shape generated by $S$ by a factor of 2 parallel to the $x$
            axis and then dilate the resulting shape by a factor of 3.  Draw the resulting
            geometric shape and find the matrix that does these transformations as one
            linear transformation.
        \item[(b)] Dilate the shape generated by $S$ by a factor of 3 and then shear the
            resulting shape by a factor of 2 parallel to the $x$
            axis.  Draw the resulting
            geometric shape and find the matrix that does these transformations as one
            linear transformation.
        \item[(c)] Rotate the shape generated by $S$ by $60^\circ$ counterclockwise, then
            dilate the resulting shape by a factor of $-1.5$, and finally shear the shape
            by a factor of 3 parallel to the $y$ axis.  Draw the resulting
            geometric shape and find the matrix that does these transformations as one
            linear transformation.
    \end{enumerate}
\end{problem}



\subsection{Linear Transformations in Abstract Vector Spaces}
\begin{problem}\label{prob:calc_lt}
    In calculus we know of two very important linear transformations.  Let $\mathcal{V}$
    be the vector space of all real-valued functions $f$ on the interval $[a,b]$ that are
    differentiable and continuous on $[a,b]$.  Let $\mathcal{W}$ be the vector space
    $C[a,b]$ of all continuous functions on $[a,b]$.  
    \begin{itemize}
        \item The transformation $D: \mathcal{V} \to \mathcal{W}$ is defined as $D(f) =
            f'$.  That is, $D$ is the transformation that takes a derivative of a
            function.
        \item The transformation $\mathcal{I}: \mathcal{W} \to \mathcal{V}$ is defined as
            $\mathcal{I}(f) = \int_a^x f(\tau) d\tau$.  That is, $\mathcal{I}$ is the
            transformation that gives the antiderivative of a function.
    \end{itemize}
    Verify that both of these well-known transformations are indeed linear
    transformations.
\end{problem}
\solution{
Pulling scalars and sums around in integrals and derivatives is appropriate.  This is
known from calc 1 but the reason is that these are linear transformations.
}

\begin{definition}[Kernel of a Linear Transformation]
    Let $T$ be a linear transformation from the vector space $\mathcal{V}$ to the vector
    space $\mathcal{W}$.  The {\bf kernel} of T is defined as
    \[ \text{Ker}(T) = \{ \bx \in \mathcal{V} \, : \, T(\bx) = \bo \in \mathcal{W} \}. \]
    Observe that the kernel is another name for the null space.
\end{definition}


\begin{problem}
    Let $D$ be the linear transformation defined as $D(f) = f'$ as in problem
    \ref{prob:calc_lt}.  What is the kernel of $D$?
\end{problem}
\solution{
The set of all constant functions.
}


% \begin{problem}
%     Let $\mathcal{I}$ be the linear transformation defined as $\mathcal{I}(f) = \int_a^x
%     f(\tau) d\tau$ as in problem \ref{prob:calc_lt}.  What is the kernel of $\mathcal{I}$?
% \end{problem}
% \solution{
% We need to get the zero function out so the only thing in the kernel is the zero function.
% }


\begin{problem}
    Define $T(y)$ on $\mathcal{V} = \{y(t) : y'(t) \text{ and } y''(t) \text{ exist } \}$
    and define $T(y) = \frac{d^2y}{dt^2}$.  What is the kernel of $T$?
\end{problem}
\solution{
The set of all linear functions.
}

\begin{problem}
    Define $T: \mathcal{P}_2 \to \mathbb{R}^2$ by 
    \[ T(p) = \begin{pmatrix} p(0) \\ p(1) \end{pmatrix}. \]
    For instance, if $p(t) = 3 + 5t + 7t^2$ then $T(p) = \begin{pmatrix} 3 \\ 15
    \end{pmatrix}$.
    \begin{enumerate}
        \item[(a)] Verify that $T$ is indeed a linear transformation.
        \item[(b)] Find a polynomial $p(x) \in \mathcal{P}_2$ that is in the kernel of $T$.
    \end{enumerate}
\end{problem}
\solution{
    (adapted from 4.2 problem 31 of \cite{Lay}). \\
    \begin{enumerate}
        \item[(a)] $T(c_1 p+c_2 q) = \begin{pmatrix} c_1 p(0) + c_2 q(0) \\ c_1 p(1) + c_2
                q(1) \end{pmatrix} = c_1 \begin{pmatrix} p(0) \\ q(0) \end{pmatrix} + c_2
            \begin{pmatrix} p(1) \\ q(1) \end{pmatrix} = c_1 T(p) + c_2 T(q) $
        \item[(b)] $p(x) = cx-cx^2$ would work just fine.
    \end{enumerate}
}

\begin{problem}
    Let $M_{2\times2}$ be the vector space of all $2 \times 2$ matrices and define $T(A) =
    A + A^T$ for $A \in M_{2 \times 2}$.  Let $B$ be any matrix in $M_{2\times2}$ such
    that $B^T = B$.  Find a matrix $A$ such that $T(A) = B$.  Then describe the kernel of
    $T$.
\end{problem}
\solution{(adapted from 4.2 problem 33 of \cite{Lay}) \\
$A = (1/2) B$\\
The kernel of $T$ consists of matrices of the form $\begin{pmatrix} 0 & a \\ -a & 0
\end{pmatrix}$ where $a \in \mathbb{R}$.
}


\begin{example}
    Determine if the transformation $T: \mathbb{R}^2 \to \mathbb{R}^2$ defined by 
    \[ T\begin{pmatrix} x_1 \\ x_2 \end{pmatrix} = \begin{pmatrix} 4x_1 - 2x_2 \\ 3 |x_2|
    \end{pmatrix} \]
    is or is not a linear transformation.\\
    {\bf Solution:} By the definition of a linear transformation we need to see check that
    $T(\bu + \bv) = T(\bu+\bv)$ and that $T(c\bu) = cT(\bu)$ for arbitrary vectors
    $\bu,\bv\in\mathbb{R}^2$.  

    Let $\bu = \begin{pmatrix} u_1 \\ u_2 \end{pmatrix}$ and $\bv = \begin{pmatrix} v_1 \\
        v_2 \end{pmatrix}$ and observe that 
    \[ T(\bu + \bv) = T\left(\begin{pmatrix} u_1 \\ u_2 \end{pmatrix} + \begin{pmatrix} v_1
        \\ v_2 \end{pmatrix} \right) = T\left( \begin{pmatrix} u_1 + v_1 \\ u_2 + v_2
\end{pmatrix} \right) = \begin{pmatrix} 4(u_1+v_1) - 2(u_2 + v_1) \\ 3 |u_2+v_1|
\end{pmatrix}. \]
    We can clearly separate the first component but due to the absolute value in the
    second component we cannot separate the result of the previous equation to form
    $T(\bu)+T(\bv)$.  Therefore, $T$ is not a linear transformation.
\end{example}

\begin{thm}
    If $T$ is a linear transformation then $T(\bo) = \bo$.
\end{thm}
\begin{proof}
    If $T$ is a linear transformation then $T(c\bu) = cT(\bu)$ for any vector $\bu$ and
    any scalar $c\in\mathbb{R}$.  If we take $c=0$ then $T(0\bu) = cT(\bu)$ which implies
that $T(\bo) = \bo$.
\end{proof}

\begin{example}
   Determine if the transformation $T(x_1,x_2,x_3) = (1,x_2,x_3)$ is a linear
   transformation. \\
   {\bf Solution:} Observe that $T(0,0,0) = (1,0,0)$ so by the previous theorem we see that
   $T$ is not a linear transformation.
\end{example}

\begin{example}
    Determine if the transformation $T(x_1,x_2,x_3) = (x_1,0,x_3)$ is a linear
    transformation. \\
    {\bf Solution:} Since $T(\bo) =\bo$ it is possible that $T$ is a linear transformation
    but we cannot use this to prove that $T$ \underline{is} linear.  We need to check that
    $T(c_1 \bu + c_2 \bv) = c_1 T(\bu) + c_2 T(\bv)$.  Indeed, let $\bu = (u_1,u_2,u_3)$
    and let $\bv = (v_1,v_2,v_3)$ and let $c_1, c_2 \in \mathbb{R}$.  Therefore,
    \[ T(c_1 \bu + c_2 \bv) = T( (c_1 u_1, c_1 u_2, c_1 u_3) + (c_2 v_1, c_2 v_2, c_2
    v_3)) = T( (c_1 u_1 + c_2 v_1, c_1 u_2 + c_2 v_2, c_1 u_3 + c_2 v_3)) \]
    Applying the transformation gives
    \[ T(c_1 \bu + c_2 \bv) = (c_1 u_1 + c_2 v_1, 0, c_1 u_3 + c_2 v_3) =\cdots =  c_1 T(\bu) + c_2
    T(\bv) \]
    which means that $T$ is indeed a linear transformation.
\end{example}

In this class we have studied two particular types of questions: solving first order
non-homogeneous differential equations and solving systems of equations.  Let's consider
the processes for these two problems side by side so that we can truly see them as the
exact same problem in the language of linear transformations.

\begin{minipage}{0.45\columnwidth}
    {\bf Non-homogeneous 1$^{st}$ order ODE}
    \begin{enumerate}
        \item Solve $y' + Py = Q(t)$
        \item Let $T(y) = y'+Py$.  We want to find $y$ so that $T(y) = Q$.
        \item Find $y_h \in \text{Ker}(T)$
        \item Find a particular $y_p$ so that $T(y_p) = Q$.
        \item The solution to $T(y) = Q$ is $y = y_h + y_p$
    \end{enumerate}
\end{minipage}
\begin{minipage}{0.45\columnwidth}
    {\bf Non-homogeneous linear system} 
    \begin{enumerate}
        \item Solve $A \bx = \bb$
        \item Let $T(\bx) = A\bx$.  We want to find $\bx$ so that $T(\bx) = \bb$.
        \item Find $\bx_h \in \text{Null}(A)$
        \item Find a particular $\bx_p$ so that $T(\bx) = \bb$.
        \item The solution to $T(\bx)=\bb$ is $\bx = \bx_h+\bx_p$
    \end{enumerate}
\end{minipage}

\begin{example}
    In this example we will solve two problems related to linear transformations.
    Let $T_1(y) = y'+0.5y$ and $Q(t) = 3$.  Let $T_2(\bx) = \begin{pmatrix} 1 & 3 \\ 2 &
        6 \end{pmatrix} \bx$ and let $\bb = \begin{pmatrix} 5 \\ 10 \end{pmatrix}$.  Solve
            $T_1(y) = Q$ and $T_2(\bx) = \bb$.

\begin{minipage}{0.45\columnwidth}
    {\bf $T_1(y) = Q$}
    \begin{enumerate}
        \item The homogeneous solution is $y_{h} \in \text{span}\{e^{-0.5t}\}$.
        \item The non-homogeneity if a constant function so $y_p \in \text{span}\{1\}$
        \item The solution to $T_1(y) = Q$ is $y = C_0 e^{-0.5t} + C_1$ where $C_0, C_1 \in
            \mathbb{R}$.
    \end{enumerate}
\end{minipage}
\begin{minipage}{0.45\columnwidth}
    {\bf $T_2(\bx) = \bb$}
    \begin{enumerate}
        \item After row reducing the homogeneous solution is $\bx_{h} \in
            \text{span}\left\{ \begin{pmatrix} -3 \\ 1
            \end{pmatrix} \right\}$
        \item After row reducing with $\bb$ on the right we see that the particular
            solution is $\bx_p = \begin{pmatrix} 5 \\ 0 \end{pmatrix}$
            \item The solution to $T_2(\bx)=\bb$ is $\bx = \begin{pmatrix} 5 \\
                    0\end{pmatrix} + \begin{pmatrix} -3 \\ 1 \end{pmatrix} t$ where $t \in
                        \mathbb{R}$.
    \end{enumerate}
\end{minipage}
\end{example}



\begin{example}
    Consider the homogeneous linear differential equation $y' + 0.5 y = 0$.  We can see this as a
    question about the kernel of a linear transformation.  Indeed, if we let $T(y) = y' -
    0.5y$ be a transformation from the space of differentiable functions to the space of
    continuous functions (on appropriate domains) then the differential equation can
    simply be stated as: find $y$ in the kernel of the transformation $T(y) = y' + 0.5y$. 

    The kernel of this linear transformation is spanned by $y(t) = e^{-0.5t}$ since $T(y)
    = 0$.  Therefore the solution to the differential equation is $y(t) = C e^{-0.5t}$.
\end{example}

\begin{problem}
    Consider the homogeneous linear differential equation $y'' + y' - y = 0$.  Rewrite this
    differential equation as a question about the kernel of an appropriate linear
    transformation.
\end{problem}
\solution{
$T(y) = y'' + y' - y$
}

\begin{problem}
    For non-homogeneous linear differential equations we can re-frame them in the language
    of linear transformations in the following way.  
    \begin{itemize}
        \item Find a function in the kernel of the transformation
        \item Find a particular solution that satisfies the non-homogeneous equation
        \item The general solution is a linear combination of the kernel solution and the
            particular solution.
    \end{itemize}
    Use this idea to solve $T(y) = \sin(t)$ where $T(y) = y' + y$
\end{problem}
\solution{
    The kernel of the linear transformation is spanned by $y(t) = e^{-t}$.  Therefore the
    solution is $C_1 e^{-t} + C_1 \sin(t) + C_2 \cos(t)$.
}

\begin{problem}
    Let $T$ be a linear transformation that maps vectors in $\mathbb{R}^2$ to vectors in
    $\mathbb{R}^2$.  Symbolically we write $T: \mathbb{R}^2 \to \mathbb{R}^2$.  Assume
    that 
    \[ T\begin{pmatrix} 1 \\ 0 \end{pmatrix} = \begin{pmatrix} 3 \\ -1 \end{pmatrix} \quad
            \text{and} \quad T\begin{pmatrix} 0 \\ 1 \end{pmatrix} = \begin{pmatrix} -7 \\
                -3 \end{pmatrix}. \]
    Use the following hints to determine the action of $T$ on an arbitrary vector
    $\begin{pmatrix} x_1 \\ x_2 \end{pmatrix} \in \mathbb{R}^2$.
        \begin{itemize}
            \item Expand $\begin{pmatrix}x_1\\x_2\end{pmatrix}$ as a linear combination of
                    the basis vectors $\begin{pmatrix}1\\0\end{pmatrix}$ and
                        $\begin{pmatrix}0\\1\end{pmatrix}$.
            \item Recall that if $T$ is a linear transformation then $T(c \bu) = cT(\bu)$ and
                $T(\bu+\bv) = T(\bu) + T(\bv)$.  Use this fact to write
                $T\begin{pmatrix}x_1\\x_2\end{pmatrix}$
            \item Simplify your answer to give the definition of $T$.
        \end{itemize}
\end{problem}
\solution{
    \begin{flalign*}
        T\begin{pmatrix} x_1 \\ x_2 \end{pmatrix} &= T\left( x_1 \begin{pmatrix}
            1\\0\end{pmatrix} + x_2 \begin{pmatrix} 0 \\ 1 \end{pmatrix} \right) \\
        &= x_1 T\begin{pmatrix} 1\\0\end{pmatrix} + x_2 T\begin{pmatrix}0\\1\end{pmatrix}
            \\
        &= x_1 \begin{pmatrix} 3\\-1\end{pmatrix}  + x_2 \begin{pmatrix} -7 \\
            -3\end{pmatrix} \\
        &= \begin{pmatrix} 3x_1 - 7x_2 \\ -x_1 - 3 x_2 \end{pmatrix}
    \end{flalign*}
}

\begin{thm}\label{thm:transformation_to_basis}
    Let $T: \mathcal{V} \to \mathcal{W}$ be a linear transformation from a vector space
    $\mathcal{V}$ to a vector space $\mathcal{W}$.  The action of $T$ on any vector $\bv
    \in \mathcal{V}$ is completely determined by the actions of $T$ on the basis vectors
    for $\mathcal{V}$.  

    More clearly:\\
    Let $\mathcal{B}=\{\bv_1,\bv_2,\ldots,\bv_k\}$ be a basis for the vector space
    $\mathcal{V}$.  Let $T$ be a linear transformation from $\mathcal{V}$ to vector space
    $\mathcal{W}$ and assume that 
    \[ T(\bv_1) = \bw_1, \quad T(\bv_2) = \bw_2, \quad \ldots \quad T(\bv_k) = \bw_k \]
    where $\bw_1, \bw_2, \ldots, \bw_k \in \mathcal{W}$.  If $\bv$ is written as a linear
    combination of basis vectors from $\mathcal{B}$
    \[ \bv = \sum_{j=1}^k c_j
        \bv_j, \]
    then 
    \[ T(\bv) = \sum_{j=1}^k c_j \bw_j \]
\end{thm}
\begin{proof}
    The proof follows from the definition of a linear transformation.
    \begin{flalign*}
        T(\bv) &= T\left( c_1 \bv_1 + c_2 \bv_2 + \cdots + c_k \bv_k \right) \\
        &= T(c_1 \bv_1) + T(c_2 \bv_2) + \cdots + T(c_k \bv_k) \\
        &= c_1 T(\bv_1) + c_2 T(\bv_2) + \cdots + c_k T(\bv_k) \\
        &= c_1 \bw_1 + c_2 \bw_2 + \cdots + c_k \bw_k
    \end{flalign*}
\end{proof}
The consequence of Theorem \ref{thm:transformation_to_basis} is that all we really need to
know is the action of a linear transformation on the basis vectors and we know the entire
definition of the transformation.\footnote{Note here that we are implicitly assuming that
    the vector spaces $\mathcal{V}$ and $\mathcal{W}$ are finite dimensional.  If they
    were infinite dimensional the theorem will still hold under suitable convergence
conditions.}

\begin{problem}
    Let $T$ be a linear transformation mapping quadratic polynomials to $2\times 2$
    matrices: $T: \mathcal{P}_2 \to M_{2\times2}$.  Recall that the set $\mathcal{B} = \{ 1, x ,x^2\}$
    is a basis for $\mathcal{P}_2$.  If 
    \begin{flalign*}
        T(1) &= \begin{pmatrix} 1 & 0 \\ 0 & 0 \end{pmatrix} \\
        T(x) &= \begin{pmatrix} 0 & 1 \\ 1 & 0 \end{pmatrix} \\
        T(x^2) &= \begin{pmatrix} 0 & 0\\ 0 & 2 \end{pmatrix} 
    \end{flalign*}
    then what is the action of $T$ on the generic quadratic polynomial $T(ax^2 + bx+c)$?
\end{problem}
\solution{
    \begin{flalign*}
        T(ax^2+bx+c)&= aT(x^2) + bT(x) + cT(1) \\
        &= a \begin{pmatrix} 0 & 0 \\ 0 & 2 \end{pmatrix} + b \begin{pmatrix} 0 & 1 \\ 1 &
            0\end{pmatrix} + c \begin{pmatrix} 1 & 0 \\ 0 & 0 \end{pmatrix} \\
        &= \begin{pmatrix} c & b \\ b & 2a \end{pmatrix}
    \end{flalign*}
}



\newpage\section{Additional Exercises}


\begin{problem}[Legendre Polynomials]
   Consider the vector space  
   \[ \mathcal{V} = \{a_0 + a_1 x + a_2 x^2 \, : \, a_0, a_1, a_2 \in
       \mathbb{R} \text{ and } x \in [-1,1] \} \]
    along with the inner product 
    \[ \left< f,g\right> = \int_{-1}^1 f(x) g(x) dx \]
    Consider the basis $\mathcal{B} = \{1,x,\frac{1}{2}(3x^2-1)\}$.  
    \begin{enumerate}
        \item[(a)] Is this basis an orthogonal basis?
        \item[(b)] Set up the necessary calculus to write $h(x) = 3x^2+2$ as a linear
            combination of vectors in $\mathcal{B}$.
    \end{enumerate}
\end{problem}
\solution{This is an orthogonal basis.  We need to consider that
    \[ c_1 (1) + c_2 ( x) + c_3 ( \frac{1}{2} (3x^2-1) ) = h(x) \]
    Since this is an orthogonal basis we can get each $c_j$ by doing
    inner products:
    \begin{flalign*}
        c_1 &= \frac{\left<h(x),1\right>}{\left<1,1\right>}=3 \\
        c_2 &= \frac{\left<h(x),x\right>}{\left<x,x\right>}=0 \\
        c_3 &=
        \frac{\left<h(x),\frac{1}{2}(3x^2-1)\right>}{\left<\frac{1}{2}(3x^2-1),\frac{1}{2}(3x^2-1)\right>}=2 \\
    \end{flalign*}
}                    


\begin{problem}
        The set $S$ below contains three linearly independent mutually orthogonal vectors
        and as such we know that $\text{span}(S) = \mathbb{R}^3$
        \[ S = \left\{ \begin{pmatrix} 3 \\ 1 \\ 1 \end{pmatrix} \, , \, \begin{pmatrix}
                -1 \\ 2 \\ 1 \end{pmatrix} \, , \, \begin{pmatrix} -1 \\ -4 \\ 7
        \end{pmatrix} \right\} \]
        Write the vector $\bu = \begin{pmatrix} 3 \\ 0 \\ 5 \end{pmatrix}$ as a linear
        combination of the vectors in $S$.
\end{problem}
\solution{
    \[ \bu = c_1 \bv_1 + c_2 \bv_2 + c_3 \bv_3 \]
    where $\bv_1, \bv_2, \bv_3 \in S$.  Since the vectors in $S$ are mutually
    orthogonal we have
    \begin{flalign*}
        c_1 &= \frac{\bu \cdot \bv_1 }{\bv_1 \cdot \bv_1} = \frac{14}{11} \\
        c_2 &= \frac{\bu\cdot \bv_2}{\bv_2 \cdot \bv_2} = \frac{2}{6} =
        \frac{1}{3} \\
        c_3 &= \frac{\bu\cdot \bv_3}{\bv_3 \cdot \bv_3} = \frac{32}{66}
        =\frac{16}{33}
    \end{flalign*}
    Therefore
    \[ \bu = \frac{14}{11} \begin{pmatrix} 3 \\ 1 \\ 1 \end{pmatrix} +
            \frac{1}{3} \begin{pmatrix} -1 \\ 2 \\ 1 \end{pmatrix} +
                \frac{16}{33} \begin{pmatrix} -1 \\ -4 \\ 7 \end{pmatrix} \]
                }

\begin{problem}
Let $\mathcal{V} = C[-1,1]$ be the vector space of continuous functions on the closed
interval $[-1,1]$.  An inner product on this vector space is $\left< f,g\right> =
\int_{-1}^1 f(x) g(x) dx$.  

        Let $f(x) = x^2 + 1$ and $g(x) = x$.  Set up (\underline{but do not evaluate}) the
        integrals to find the angle between these two functions in $C[-1,1]$.  
\end{problem}
\solution{
    \begin{flalign*}
        \| f \| &= \sqrt{ \int_{-1}^1 (x^2+1)^2 dx} \\
        \| g \| &= \sqrt{ \int_{-1}^1 x^2 dx } \\
        \left< f , g\right> &= \int_{-1}^1 (x^2+1)(x) dx \\
        \cos(\theta) &= \frac{\left< f , g \right> }{\|x^2+1\| \|x\|}
    \end{flalign*}
}

\begin{problem}
    If $A$ and $B$ are arbitrary $m \times n$ matrices, then the mapping $\left<
    A,B\right> = \text{trace}(A^TB)$ defines an inner product in $\mathbb{R}^{m\times n}$.
    Use this inner product to find $\left<A,B\right>$, the norms, $\|A\|$ and $\|B\|$, and
    the angle between $A$ and $B$ for
    \[ A = \begin{pmatrix} -2 & 3 \\ -1 & 3 \\ 1 & -2 \end{pmatrix} \quad \text{and} \quad
            B = \begin{pmatrix} -2 & 1 \\ 1 & 2 \\ 1 & 1 \end{pmatrix}. \]
\end{problem}
\solution{
    Hint: Recall that for $\bv \in \mathcal{V}$ we have $\|\bv\| = \left< \bv,\bv\right>$
    and if $\bw \in \mathcal{V}$ then the angle between $\bv$ and $\bw$ is defined as
$\left< \bv,\bw\right> = \|\bv\| \|\bw\| \cos\theta$.
}

\begin{problem}
    Find a non-trivial vector that is perpendicular to both $\bv$ and $\bu$ where
    \[ \bv = \begin{pmatrix} 2 \\ 5 \\ 3 \end{pmatrix} \quad \text{ and } \quad \bu =
            \begin{pmatrix} -1 \\ -2 \\ -2 \end{pmatrix} \]
\end{problem}
\solution{
    Hint: Use the Gram-Schmidt process.
}

\begin{problem}
    Determine which of the following transformations are linear transformations.
    \begin{flalign*}
        T_1(x,y,z) &= (x,y,-z) \\
        T_2(x,y) &= (2x-3y,x+4,5y) \\
        T_3(x,y,z) &= (1,y,-z) \\
        T_4(x,y) &= (4x-2y,3|y|) \\
        T_5(x,y,z) &= (x,0,z) 
    \end{flalign*}
\end{problem}
\solution{
    Hint: check out the exact definition of a linear transformation.
}

\begin{problem}
    Let $T: \mathbb{R}^2 \to \mathbb{R}^2$ be a linear transformation that sends the
    vector $\bu = \begin{pmatrix} 5\\2\end{pmatrix}$ to $\begin{pmatrix}
        2\\1\end{pmatrix}$ and maps $\bv = \begin{pmatrix}1\\3\end{pmatrix}$ to
            $\begin{pmatrix} -1\\3\end{pmatrix}$.  Use properties of linear
                transformations to calculate the following.
    \begin{enumerate}
        \item[(a)] $T(-3\bu)$
        \item[(b)] $T(8\bv)$
        \item[(c)] $T(-3\bu + 8\bv)$
    \end{enumerate}
\end{problem}
\solution{
    Hint: recall that in a linear transformation, $T(a\bv) = aT(\bv)$ and $T(\bv+\bu) =
    T(\bv) + T(\bu)$.
}


\begin{problem}
    Let $e_1 = \begin{pmatrix}1\\0\end{pmatrix}$ and $e_2 =
        \begin{pmatrix}0\\1\end{pmatrix}$.  Let $\bv = \begin{pmatrix}-3\\9\end{pmatrix}$ and let $\bu =
            \begin{pmatrix}-5\\-8\end{pmatrix}$.
    If $T:\mathbb{R}^2 \to \mathbb{R}^2$ is a linear transformation that sends $e_1$ to
    $\bv$ and $e_2$ to $\bu$.  Where does $T$ send $\begin{pmatrix}-4\\6\end{pmatrix}$?
\end{problem}
\solution{
    Hint: Start by writing $(-4,6)$ as a linear combination of $e_1$ and $e_2$.
}


\begin{problem}
    Let $T:\mathcal{P}_2 \to \mathcal{P}_2$ be a linear transformation mapping quadratic
    functions to quadratic functions.  Assume that 
    \[ T(1) = 4x^2 + 4, \quad T(x) = 4x+3, \quad \text{and} \quad T(x^2) = 4x^2 + x + 3.
        \]
    Find the image of an arbitrary quadratic polynomial $ax^2 + bx + c$ under $T$.  That
    is, find $T(ax^2 + bx + c)$.
\end{problem}
\solution{
    Hint: Apply the fact that $T$ is a linear transformation to $T(ax^2 + bx + c)$ to
    first get 
    \[ T(ax^2 + bx + c) = aT(x^) + bT(x) + cT(1). \]
    Then apply the transformation given above and simplify.
}




\begin{problem}
    Assume that $T: \mathbb{R}^2 \to \mathbb{R}^2$ is a linear transformation and $T(\bv)
    = \begin{pmatrix} 2&0\\0&1\end{pmatrix}\begin{pmatrix}0&1\\-1&0\end{pmatrix} \bv$.
    \begin{enumerate}
        \item[(a)] Which of the following best describes what $T$ does to the plane?
            \begin{itemize}
                \item $T$ stretches vectors vertically by a factor of 2, then rotates them
                    $90^\circ$ clockwise.
                \item $T$ rotates vectors $90^\circ$ clockwise, then stretches them
                    vertically by a factor of 2.
                \item $T$ stretches vectors horizontally by a factor of 2, then rotates them
                    $90^\circ$ clockwise.
                \item $T$ rotates vectors $90^\circ$ clockwise, then stretches them
                    horizontally by a factor of 2.
            \end{itemize}
        \item[(b)] Explain your choices from part (a)
    \end{enumerate}
\end{problem}