\documentclass[11pt]{article}

\usepackage{fullpage}
\usepackage{graphicx,epsfig}


\begin{document}

\title{{\bf Dynamic Programming: Matrix chain multiplication}\\
\normalsize{(CLRS 15.2)}}


%\center{There is no intellectual exercise which is not, eventually, useless. 
%from Pierre Menard, Author of Don Quixote by Jorge Luis Borges}


\date{}
\maketitle

\vspace{-1cm}
\section{The problem} 
Given a sequence of matrices $A_1,A_2,A_3,... ,A_n$, find the best way
(using the minimal number of multiplications) to compute their
product.
\begin{itemize}
\item Isn't there only one way? $((\cdots ((A_1 \cdot A_2) \cdot
  A_3)\cdots ) \cdot A_n)$
\item No, matrix multiplication is {\em associative}. \\
  e.g. $A_1 \cdot (A_2 \cdot (A_3 \cdot (\cdots (A_{n-1} \cdot
  A_n)\cdots )))$ yields the same matrix.
\item Different multiplication orders do not cost the same:
  \begin{itemize}
  \item Multiplying $p \times q$ matrix $A$ and $q \times r$
    matrix $B$ takes $p \cdot q \cdot r$ multiplications;
    result is a $p \times r$ matrix.
  \item Consider multiplying $10 \times 100$ matrix $A_1$
    with $100 \times 5$ matrix $A_2$ and $5 \times 50$ matrix
    $A_3$. 
    \begin{itemize}
    \item[--] $(A_1 \cdot A_2) \cdot A_3$ takes $10
      \cdot 100 \cdot 5 + 10 \cdot 5 \cdot 50 = 7500$ multiplications.
    \item[--] $A_1 \cdot (A_2 \cdot A_3)$ takes $100
      \cdot 5 \cdot 50 + 10 \cdot 50 \cdot 100 = 75000$
      multiplications.
    \end{itemize}
  \end{itemize}
\end{itemize}

\section{Notation} 

\begin{itemize}  
\item In general, let $A_i$ be $p_{i-1} \times p_i$ matrix.
  
\item Let $m(i,j)$ denote minimal number of multiplications needed to
  compute $A_i\cdot A_{i+1}\cdots A_j$
\item We want to compute $m(1,n)$.

\end{itemize}


\section{Recursive algorithm}
\begin{itemize}
\item Assume that someone tells us the position of the {\bf last}
  product, say $k$. Then we have to compute recursively the best way
  to multiply the chain from $i$ to $k$, and from $k+1$ to $j$, and
  add the cost of the final product.  This means that
  
  $$m(i,j) = m(i,k) + m(k+1,j) + p_{i-1} \cdot p_k \cdot p_j$$
  
\item If noone tells us $k$, then we have to try all possible values
  of $k$ and pick the best solution.

\item Recursive formulation  of $m(i,j)$: \\
  \[ m(i,j) = \left\{ \begin{array}{ll}
      0                      & \mbox{ If $i=j$} \\
      \min_{i \leq k < j} \{m(i,k) + m(k+1,j) + p_{i-1} \cdot p_k \cdot p_j \}   & \mbox{ If $i<j$} \\
    \end{array}
  \right. \]
\item To go from the recursive formulation above to a program is pretty straightforward: \\ \\
  \fbox{
\parbox{14cm}{
{\sc Matrix-chain$(i,j)$}
\begin{itemize}
\item[] IF $i=j$ THEN return 0 
\item[] $m = \infty$ 
\item[] FOR $k=i$ TO $j-1$ DO
  \begin{itemize}
  \item[] q = {\sc Matrix-chain$(i,k)$} $+$ {\sc Matrix-chain$(k+1,j)$} $+ p_{i-1} \cdot p_k \cdot p_j$ 
  \item[] IF $q < m$ THEN $m = q$ 
  \end{itemize}
\item[] OD 
\item[] Return $m$
\end{itemize}
END {\sc Matrix-chain} \\ 

Return {\sc Matrix-chain}$(1,n)$
}} 
\item Running time:
        \begin{eqnarray*}
    T(n) & = & \sum_{k=1}^{n-1} (T(k) + T(n-k) + O(1)) \\
          & = & 2 \cdot \sum_{k=1}^{n-1} T(k) + O(n) \\
    & \geq & 2\cdot T(n-1) \\
    & \geq & 2\cdot 2\cdot T(n-2) \\
    & \geq & 2\cdot 2\cdot 2\dots \\
    & = & 2^n
        \end{eqnarray*}
\item  Exponential is ... SLOW!
%   \begin{itemize}
%   \item Guess solution: $T(n) \geq 2^{n-1}$
%   \item Test:
%   \begin{eqnarray*}
%   T(n) & = & 2 \cdot \sum_{k=1}^{n-1} T(n) + O(n) \\
%        & \geq & 2 \cdot \sum_{k=1}^{n-1} 2^{k-1} + O(n) \\
%        & = & 2 \cdot  \sum_{k=0}^{n-2} 2^k +O(n) \\
%        & = & 2(\frac{2^{n-1}-1}{2-1}+1) + O(n) \\
%        & = & 2^n + O(n) \\
%        & \geq & 2^{n-1} \\
%   \end{eqnarray*}
%   \end{itemize}
\item Problem is that we compute the same result over and over again. 
  \begin{itemize}
  \item Example: Recursion tree for {\sc Matrix-chain$(1,4)$}
    
    \centerline{\includegraphics[height=4cm]{recursion_tree.pdf}} 
    
    For example, we compute {\sc Matrix-chain$(3,4)$} twice.
  \end{itemize}
\end{itemize}  


\section{Dynamic programming with a table and recursion} 
\begin{itemize} 
\item Solution is to ``remember" the values we have already computed in a
table.  This is called {\em memoization}.  We'll have a table T[1..n][1..n] such that T[i][j] stores the solution to problem Matrix-CHAIN(i,j). Initially all entries will be set to $\infty$. 

\fbox{
  \parbox{15cm}{
    FOR $i=1$ to $n$ DO
    \begin{itemize}
    \item[] FOR $j=i$ to $n$ DO
      \begin{itemize}
      \item[] $T[i][j] = \infty$
      \end{itemize}
    \item[] OD
    \end{itemize}
    OD 
  }
}

\item The code for MATRIX-CHAIN(i,j) stays the same, except that it
  now uses the table.  The first thing MATRIX-CHAIN(i,j) does is to
  check the table to see if $T[i][j]$ is already computed. Is so, it
  returns it, otherwise, it computes it and writes it in the
  table. Below is the updated code.
  
  \fbox{
    \parbox{15cm}{
      {\sc Matrix-chain$(i,j)$}
      \begin{itemize}
      \item[] IF $T[i][j] < \infty$ THEN return $T[i][j]$
      \item[] IF $i=j$ THEN $T[i][j] = 0$, return 0 
      \item[] $m=\infty$
      \item[] FOR $k=i$ to $j-1$ DO
        \begin{itemize}
        \item[] q = {\sc Matrix-chain$(i,k)$} + {\sc Matrix-chain$(k+1,j)$}$ + p_{i-1} \cdot p_k \cdot p_j$
        \item[] IF $q < m$ THEN $m = q$
        \end{itemize}
      \item[] OD 
      \item [] $T[i][j] = m$
      \item[] return $m$ 
      \end{itemize}
      END {\sc Matrix-chain} \\ 
      
      
      return {\sc Matrix-chain$(1,n)$}
    }
  }
  
\item The table will prevent a subproblem
  MATRIX-CHAIN(i,j) to be computed more than once. 
\item Running time: 
  \begin{itemize}
  \item $\Theta(n^2)$ different calls to {\sc matrix-chain$(i,j)$}.
  \item The first time a call is made it takes $O(n)$ time, {\em not}
    counting recursive calls.
  \item When a call has been made once it costs $O(1)$ time to make
    it again.\\
    $\Downarrow$ 
    
    $O(n^3)$ time
  \item Another way of thinking about it: $\Theta(n^2)$ total entries
    to fill, it takes $O(n)$ to fill one.
  \end{itemize}
\end{itemize}


\section{Dynamic Programming without recursion}

\begin{itemize}
\item Often dynamic programming is presented as filling up a table
  from the bottom, in such a way that makes recursion unnecessary.
  Avoiding recursion is perhaps elegant, and necessary 20-30 years ago
  when programming languages did not include support for recursion.

\item Personally, I like top-down recursive thinking, and I think compilers are pretty effective at optimizing recursion. 

\item Solution for Matrix-chain without recursion: Key is that
  $m(i,j)$ only depends on $m(i,k)$ and $m(k+1,j)$ where $i \leq k < j
  \Rightarrow$ if we have computed them, we can compute $m(i,j)$
  \begin{itemize}
  \item We can easily compute $m(i,i)$ for all $1 \leq i \leq n$
  $(m(i,i)=0)$
  \item Then we can easily compute $m(i,i+1)$ for all $1 \leq i \leq
  n-1$ \\
  $m(i,i+1)=m(i,i)+m(i+1,i+1)+p_{i-1}\cdot p_i\cdot p_{i+1}$
  \item Then we can compute $m(i,i+2)$ for all $1 \leq i \leq n-2 \\ 
  m(i,i+2) = \min \{ m(i,i)+m(i+1,i+2)+p_{i-1} \cdot p_i \cdot
  p_{i+2}, m(i,i+1)+m(i+2,i+2)+p_{i-1} \cdot p_{i+1} \cdot p_{i+2}\}$

  $\vdots$ 
  \item Until we compute $m(1,n)$
  \item Computation order:

  \centerline{\includegraphics[height=4cm]{table.pdf}} 
  \end{itemize}


\item Program:
 
\fbox{
\parbox{9.5cm}{
  FOR $i=1$ to $n$ DO
    \begin{itemize}
    \item[] $T[i][i]=0$
    \end{itemize}
  OD \\
  FOR $l=1$ to $n-1$ DO
    \begin{itemize}
    \item[] FOR $i=1$ to $n-l$ DO
      \begin{itemize}
      \item[] $j=i+l$
      \item[] $T[i][j] = \infty$
      \item[] FOR $k=1$ to $j-1$ DO
        \begin{itemize}
        \item[~~] $q = T[i][k] + T[k+1][j] +
        p_{i-1}\cdot p_k\cdot p_j$
        \item[~~] IF $q < T[i[j]$ THEN $T[i][j] = q$
        \end{itemize}
      \item[] OD
      \end{itemize}
    \item[] OD
    \end{itemize}
    OD
  }}
\item Analysis:  $O(n^2)$ entries, $O(n)$ time to compute each $\Rightarrow
  O(n^3)$.

\end{itemize}



\section{Extensions} 

Above we only computed the best way to multiply the chain (with the
smallest number of operations).  The algorithm can be extended to
compute the actual order of multiplications corresponding to this
optimal cost (we'll do this as homework or in-class exercise).

This is a simplification that is often done: focus on computing the
optimal cost, and leave the details of computing the solution
corresponding to that optimal cost for later.




\end{document}
