\documentclass[a4paper]{article}
\usepackage{beamerarticle}
\mode<article>{
\usepackage{fullpage}
\usepackage{hyperref}
}
%
% OR
%
%\documentclass[handout]{beamer}
%
% OR
%
%\documentclass{beamer}
\mode<presentation>
{
  \usetheme[compress]{Rochester}
  \setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{times}
\usepackage[T1]{fontenc}
\usepackage{pict2e}
\usepackage{colortbl}

\title{Queue systems}
\subtitle{and how to use TORQUE \& Maui}

\author{Piero~Calucci}
\institute[SISSA]{Scuola Internazionale Superiore di Studi Avanzati\\ Trieste}
\date{March 9th 2007\\Advanced School\\in High Performance Computing Tools\\for e-Science}

\mode<presentation>{\subject{Cluster Resource Management slides}}
\mode<article>{\subject{Cluster Resource Management notes}}
\keywords{TORQUE Maui cluster resource RM scheduling}

\mode<article>{\newcommand{\ar}[1]{\emph{\small #1}}}
% This is only inserted into the PDF information catalog. Can be left out.


% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

% \pgfdeclareimage[height=0.5cm]{university-logo}{university-logo-filename}
% \logo{\pgfuseimage{university-logo}}



% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
%\AtBeginSubsection[]
%{
%  \begin{frame}<beamer>
%    \frametitle{Outline}
%    \tableofcontents[currentsection,currentsubsection]
%  \end{frame}
%}


% If you wish to uncover everything in a step-wise fashion, uncomment
% the following command: 

%\beamerdefaultoverlayspecification{<+->}


\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\mode<presentation>{
\begin{frame}
\frametitle{Outline}
  \tableofcontents
  % You might wish to add the option [pausesections]
\end{frame}
}

\section[The Problem]{The Problem We Are Trying to Solve}

\begin{frame}
\frametitle{The User's Problem}
\begin{itemize}
\item<1-> have \alert{dedicated} resources\\\emph{ multitasking is Bad for HPC}
\item<2-> have resources as soon as possible\\\emph{ you need to have your computation done by next week, right?}
\item<3-> have jobs run unattended and results delivered back to you\\\emph{ what do you want to do at 4.30AM?}
\end{itemize}

\mode<article>{\ar{Here I hope to show you that working in a batch system is not only more trouble for you, it also has definite advantages.}}

\end{frame}

\begin{frame}
\frametitle{The Admin's Problem}
\begin{itemize}
\item<1-> minimize resource waste
\item<2-> promote fair share of resources\\ \emph{a.k.a. <<avoid complaints from users>>}
\item<3-> monitor and account for everything
\end{itemize}

\mode<article>{\ar{We are not here to learn Resource Management internals. However, having an idea of what is going on may be useful to understand why those Nasty System Administrators are imposing so many limits\ldots}}
\end{frame}


\section[TORQUE]{Using the Resource Manager}

\begin{frame}
\frametitle{The Resource Manager}

At the core of a batch system there is a RM that:

\begin{itemize}
\item accepts \alert{job submissions} from users
\item<2-> tracks resource usage
\item<3-> delivers jobs to \alert{execution} nodes
\item<4-> informs users about job status
\end{itemize}

\end{frame}


\begin{frame}
\frametitle{The TORQUE Resource Manager}
\uncover<2->{The }\textbf{T}erascale \textbf{O}pen-source \textbf{R}esource and \textbf{QUE}ue manager \uncover<2->{is deployed as}

\begin{itemize}
\item<2-> a \alert{server component} (\texttt{pbs\_server}) on the masternode
\item<3-> an \alert{execution mini-server} (\texttt{pbs\_mom}) on each \mbox{execution node}
\end{itemize}
\uncover<4->{\small There is also a \alert{scheduler component}, but we will use the Maui Scheduler instead -- more on this later}

\mode<article>{\ar{Job scheduling can be as simple as a straight FIFO, but the use of an advanced scheduler allows much greater flexibility and better resource usage.}}

\end{frame}


\subsection*{Jobs}

\begin{frame}
\frametitle{A Job's Life}
	\begin{enumerate}
	\item<1-> a \alert{job} is a shell script that contains a description of the \alert{resources} needed and the command you want to execute
	\item<2-> you \alert{submit} the job to the batch system\\
	
	\item<3-> the batch system sends the job to an \alert{execution queue} where it is executed without human intervention
	\item<4-> job results are then delivered back to you
	\end{enumerate}
\end{frame}


\begin{frame}
\frametitle{Job Must Be a Shell Script}
	\begin{columns}[t]
	\begin{column}{0.45\textwidth}
		A \alert{job script} contains a description of the \alert{resources} you request and all the commands your job needs to perform.
		
		Resource description always comes at the beginning of the script and is identified by the \alert{\texttt{\#PBS}} mark.
		
	\end{column}
	\begin{column}{0.50\textwidth}
		{\tt\small
		\uncover<2->{\#!/bin/sh}
		
		\uncover<2->{\#PBS -l walltime=1:00:00}
		
		\uncover<2->{\#PBS -l nodes=1:ppn=2}
		
		\uncover<2->{\#PBS -N MyTestJob}
		
		\uncover<3->{do\_something\_useful}
		
		\uncover<3->{do\_more}
		
		\uncover<3->{exit 0}
		}
	\end{column}
	\end{columns}

\mode<article>{\ar{The <<do something>> part of the job script can be fairly complex. You may want to copy your data from some shared storage to local filesystems (if available) for better performance, set up the execution environment, initialize the parallel environment and copy back your results at job completion.}}
\end{frame}

\begin{frame}
\frametitle{Job Submission}

Jobs are submitted to the batch system by means of the \alert{\texttt{qsub}} command, as in

{\hspace*{4em}\tt qsub job.sh}

\uncover<2->{But you can also add resource description directly on the command line:

{\hspace*{4em}\tt qsub -l nodes=4:ppn=4 job.sh}

{\small This is especially useful when you are experimenting with subtle variations of a job submission.}
}
\end{frame}

\begin{frame}
\frametitle{Queues}
Batch systems are usually configured with multiple \alert{queues}.

Each queue can be configured to accept job from a certain group of users, or within specified resource limits, or simply on request from the user.

\alert{Be sure to select the right queue for your jobs.}

\uncover<2->{Queue selection is performed with {\tt -q queuename} on the {\tt qsub} command line or with {\tt \#PBS -q queuename} in the job script.}

\mode<article>{\ar{There can be \textbf{routing queues} and \textbf{execution queues}, and the admin can restrict <<direct>> access to routing queues only; your job is then \textbf{routed} to some execution queue by the RM.}}
\end{frame}

\begin{frame}
\frametitle{Simple Resource Specification}

\begin{tabular}{@{\tt}r@{\extracolsep{1em}\small}l}
{\tt -l nodes=\emph{n}} & request \emph{n} execution nodes\\\pause

{\tt -l nodes=\emph{n}:ppn=\emph{m}} & request \emph{n} execution nodes\\
	&with \emph{m} CPUs each\\\pause

{\tt -l walltime=\emph{n}} & request \emph{n} seconds of wallclock time\\
	& (walltime can be specified also\\
	& as \emph{hours:minutes:seconds})\\\pause
	
{\tt -l nodes=\emph{n}:\emph{feature}} & request \emph{n} nodes with a specific feature\\
	& e.g. we use {\tt :myri} \\
	& for nodes with Myrinet cards\\\pause
	
{\tt -q \emph{name}} & submit job to named queue\\\pause

{\tt -N \emph{name}} & give job a name\\
\end{tabular}

\end{frame}

\begin{frame}
\frametitle{Interactive Jobs}

If resources are available right now you can run \alert{interactive} jobs with \alert{\tt qsub -I}

In an interactive job you are given a shell on a computing node and are allowed to execute all your computation interactively, possibly on several nodes.\pause

\texttt{\small
\textbf{masternode \$} qsub -I -q smp -l walltime=5:00 -l nodes=1:ppn=2\\
qsub: job 29506.cerbero.hpc.sissa.it ready\\
\mbox{}\\
\textbf{a211 \$} 
}
\end{frame}



\section{Understanding Resource Management}

\begin{frame}
\frametitle{(No) Access to Computing Nodes}

\begin{itemize}
\item<1-> no <<normal>> user access to computing nodes
\item<2-> access permissions are created on the fly by the RM when (and where) needed for your job to run
\item<3-> while a job is running you are granted interactive access to nodes allocated to your job
\item<4-> at job completion access rights are cleared
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Node Access and Resource Limit Enforcement}

\begin{itemize}
\item<1-> access right is granted only to nodes allocated to your job\\
{\small this enforces the limit on the number of nodes you can access and guarantees that no concurrent usage of a resource is possible}
\item<2-> access right is granted only for the walltime allocated to your job\\
{\small when your allocated walltime expires, you are given a short \emph{grace time}, then all your processes on the computing node are \alert{killed}}
\item<3->you should arrange so that your jobs completes before the walltime limit, or save partial results before the job is killed
\end{itemize}

\end{frame}


%\subsection*{Job Dependencies}
%
%\begin{frame}
%\frametitle{Dependency Specification}
%
%
%
%\end{frame}
%
%\begin{frame}
%\frametitle{Beware of Deadlocks!}
%
%\end{frame}


\subsection*{TORQUE Monitoring Commands}

\begin{frame}
\frametitle{Queue Status}

\begin{tabular}{lp{0.6\textwidth}}
{\tt qstat} & query queue status\\
{\tt qstat -a} & alternate form\\\pause
{\tt qstat -r} & show only running jobs\\
{\tt qstat -rn} & only running jobs, w/ list of allocated nodes\\\pause
{\tt qstat -i} & only idle jobs\\\pause
{\tt qstat -u \emph{username}} & show jobs for named user
\end{tabular}

\end{frame}

\begin{frame}
\frametitle{Job Trace}

\begin{tabular}{l@{\extracolsep{1em}\small}l}
{\tt tracejob \emph{id}} & show what happened today to job \emph{id}\\
{\tt tracejob -n \emph{d} \emph{id}} & search last \emph{d} days
\end{tabular}

{\small searching the RM logs is a time-consuming operation, don't abuse it!}
\pause
\vspace*{2ex}

{\small\tt
\$ tracejob 29506\\
\mode<presentation>{\tiny}
Job: 29506.cerbero.hpc.sissa.it\\

02/26/2007 10:12:39  S    Job Queued at request of cxxxxx@cerbero.hpc.sissa.it, owner =
cxxxxxx@cerbero.hpc.sissa.it, job name = STDIN, queue = em64ts\\
02/26/2007 10:12:40  S    Job Modified at request of maui@cerbero.hpc.sissa.it\\
02/26/2007 10:12:40  S    Job Run at request of maui@cerbero.hpc.sissa.it\\
02/26/2007 10:12:40  S    Job Modified at request of maui@cerbero.hpc.sissa.it\\
02/26/2007 10:19:36  S    Exit\_status=265 resources\_used.cput=00:00:00 resources\_used.mem=2940kb
resources\_used.vmem=89532kb resources\_used.walltime=00:06:51
}

\end{frame}


\begin{frame}
\frametitle{The Scheduler}

The Maui Scheduler prioritizes jobs in the \alert{idle} queue, according to admin-defined policies. The highest-priority job is run as soon as resources are available.

\mode<presentation>{\vspace{\fill}}
\uncover<2->{Jobs can be \alert{blocked} if their requirements exceed available resources. Blocked jobs have an undefined priority.}

\mode<presentation>{\vspace{\fill}}
\uncover<3->{Job priorities are recomputed at each scheduler iteration, so your job can move up and down the idle queue as an effect of resource usage by other jobs of yours.}

\mode<article>{\ar{TORQUE knows nothing about priorities and blocked jobs, so the output of \texttt{qstat} doesn't tell you much about the effective placement of your job in the idle queue -- or outside it.}}

\end{frame}



\begin{frame}
\frametitle{Queues as Seen by Maui}

\mode<presentation>{\vspace*{4ex}}
{\tt\scriptsize
\$ showq

\only<1>{%
ACTIVE JOBS--------------------

\begin{tabular}{lrrrrr}
JOBNAME &          USERNAME &    STATE &PROC & REMAINING &          STARTTIME\\
&&&&&\\
29199 &               axxxx &  Running &  32 &   1:59:17 &Wed \ldots\\
29055 &            sxxxxxxx &  Running &   8 &   4:03:07 &Tue \ldots\\
28496 &            mxxxxxxx &  Running &   4 &   5:24:00 &Sat \ldots\\
&\ldots&&&&
\end{tabular}

\begin{tabular}{lrrlr}
    27 Active Jobs &   125 of &142&Processors Active&(88.03\%)\\
                   &    52 of & 58&Nodes Active     &(89.66\%)
\end{tabular}
}%only 1

\only<2>{%
IDLE JOBS----------------------

\begin{tabular}{lrrrrr}
JOBNAME &          USERNAME &    STATE &PROC &   WCLIMIT &          QUEUETIME\\
&&&&&\\
29069 &               sxxxx &     Idle &   4 &1:21:00:00 &Mon Feb 19 \ldots\\
29019 &            kxxxxxxx &     Idle &   4 &4:00:00:00 &Mon Feb 19 \ldots\\
29076 &             fxxxxxx &     Idle &   4 &4:00:00:00 &Mon Feb 19 \ldots\\
&\ldots&&&&
\end{tabular}

22 Idle Jobs

}%only 2
\only<3>{%
BLOCKED JOBS----------------

\begin{tabular}{lrrrrr}
JOBNAME &          USERNAME &    STATE &PROC &   WCLIMIT &          QUEUETIME\\
&&&&&\\
28777 &            rxxxxxxx &     Hold &   8 &2:00:00:00 &Thu Feb 15 \ldots\\
28892 &            dxxxxxxx &BatchHold &   4 &4:00:00:00 &Sat Feb 17 \ldots\\
29025 &               axxxx &     Idle &   4 &4:00:00:00 &Mon Feb 19 \ldots\\
&\ldots&&&&
\end{tabular}

Total Jobs: 71   Active Jobs: 27   \\
Idle Jobs: 22   Blocked Jobs: 22
}%only 3
}%tt
\mode<presentation>{\vspace*{\fill}}

{\tiny{(\em usernames obfuscated to protect the guilty)}}
\end{frame}

%\setlength{\unitlength}{1800sp}
%\begin{frame}[fragile]
%\frametitle{The Big Picture}
%
%\end{frame}
%
%\subsection*{pbs\_server -- maui -- pbs\_mom interaction}
%
%\subsection*{user -- pbs\_server interaction}
%
%\subsection*{pbs\_server -- job -- pbs\_mom interaction}
%
%\subsection*{pbs-provided environment for jobs}


%\subsection*{More Maui Commands}

%\begin{frame}
%\frametitle{More on Queue Status}
%
%\end{frame}

\begin{frame}
\frametitle{The Backfill Window}

\definecolor{cjob1}{rgb}{1,0.8,0}
\definecolor{cjob2}{rgb}{0.7,0.7,1}
\definecolor{cjob3}{rgb}{0.5,1,0.5}
\begin{tabular}{@{\small}lccc}
& node 1 & node 2 & node 3 \\
0:00 & \only<2->{\cellcolor{cjob1}{job1}} & \only<2->{\cellcolor{cjob1}{job1}} & \only<5->{\cellcolor{cjob3}{job3}}\\
1:00 & \only<2->{\cellcolor{cjob1}{job1}} & \only<2->{\cellcolor{cjob1}{job1}} & \only<5->{\cellcolor{cjob3}{job3}}\\
2:00 & \only<3->{\cellcolor{cjob2}{job2}} & \only<3->{\cellcolor{cjob2}{job2}} & \only<3->{\cellcolor{cjob2}{job2}}
\end{tabular}

\begin{itemize}
\item<4->\emph{job2} cannot run until \emph{job1} is done
\item<5->if you submit a \emph{job3} that requires only one node for two hours or less you can run before \emph{job2}\,!
\end{itemize}

\end{frame}


\begin{frame}
\frametitle{Discovering Free Resources}

The \alert{\tt showbf} command queries the scheduler and display resources that are available for immediate use.
\pause

\mode<presentation>{\vspace*{2ex}}
\begin{tabular}{l@{\extracolsep{1em}\small}l}
\uncover<2->{{\tt showbf} & summary of free resources}\\
\uncover<3->{{\tt showbf -f myri} & select only nodes with a given feature}\\
\uncover<4->{{\tt showbf -p intel} & select only nodes in a given partition}
\end{tabular}
\vspace*{2ex}

\begin{minipage}[t][0.3\textheight][t]{\textwidth}
{\small %
\only<2>{\texttt{%
\textbf{\$ showbf}\\
backfill window (user: 'cxxxxxx' group: 'bxxxxxxx' partition: ALL) Mon Feb 26 13:46:16\\
5 procs available with no timelimit
}}

\only<3>{\texttt{%
\textbf{\$ showbf -f myri}\\
backfill window (user: 'cxxxxxx' group: 'bxxxxxxx' partition: ALL) Mon Feb 26 13:49:16\\
no procs available
}}

\only<4>{\texttt{%
\textbf{\$ showbf -p intel}\\
backfill window (user: 'cxxxxxx' group: 'bxxxxxxx' partition: intel) Mon Feb 26 13:51:16\\
partition intel:\\
  4 procs available for 5:30:00
}}
}%small
\end{minipage}

\end{frame}


%\section[Understanding Scheduling]{Understanding Job Scheduling}

%\subsection*{Reservations}

%\begin{frame}
%\frametitle{User Reservations}
%
%\end{frame}

\mode<presentation>{
\begin{frame}
\begin{center}
\includegraphics[width=0.7\textwidth]{thatsall.png}
\end{center}

\vspace{\fill}
\begin{flushright}
\small
\url{<calucci@sissa.it>}
\end{flushright}

\end{frame}
}

\end{document}
