%\documentclass[a4paper]{article}
%\usepackage{beamerarticle}
%
% OR
%
%\documentclass[handout]{beamer}
%
% OR
%
\documentclass{beamer}

\mode<article>{%
	\usepackage{fullpage}
	\usepackage{hyperref}
} %article

\mode<presentation>
{
  \usetheme[compress]{Rochester}
  \setbeamercovered{transparent}
}

\mode<presentation>{\subject{Cluster Resource Management slides}}
\mode<article>{\subject{Cluster Resource Management notes}}
\keywords{TORQUE Maui cluster resource RM scheduling admin install}

\mode<article>{\newcommand{\ar}[1]{\emph{\small #1}}}


\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{times}
\usepackage[T1]{fontenc}
\usepackage{pict2e}
\usepackage{colortbl}
\usepackage{graphicx}

\title{Setting up Queue Systems}
\subtitle{with TORQUE \& Maui}

\author{Piero~Calucci}
\institute[SISSA]{Scuola Internazionale Superiore di Studi Avanzati\\Trieste}
\date{March 14th 2007\\Advanced School\\in High Performance Computing Tools\\for e-Science}

\begin{document}

\begin{frame}
  \titlepage
\end{frame}

\mode<presentation>{
\begin{frame}
  \frametitle{Outline}
  \tableofcontents
\end{frame}
}

\section{Obtaining and compiling TORQUE and Maui}

\subsection*{Obtaining and compiling TORQUE}

\begin{frame}
\frametitle{TORQUE Source Code}

TORQUE is available from \texttt{www.clusterresources.com}
\mode<presentation>{\vspace*{\fill}}

\begin{center}
\includegraphics[width=0.8\textwidth]{torque-download.jpg}
\end{center}
\end{frame}


\begin{frame}
\frametitle{Building TORQUE}

\begin{itemize}
\item<1-> \texttt{configure --prefix=/whatever/you/like\\make\\su\\make install}
\item<2-> not very clean, actually:\\
quite a lot of important files go into \texttt{/var/spool}\\ --- including configuration files!
\end{itemize}

\uncover<3->{You can build only the server or MOM components, just tell \texttt{-{}-disable-mom} or \texttt{-{}-disable-server}}

\uncover<4->{My favorite install uses a directory that is shared among the masternode and the computing nodes, so that I need to build only once.}

\end{frame}
\mode<article>{%
TORQUE install uses 700 permission for \texttt{pbs\_mom}, so you need to
\begin{itemize}
\item share the install directory with \texttt{no\_root\_squash}\hspace{3em}\textbf{or}
\item \texttt{chgrp nfsnobody pbs\_mom ; chmod 710 pbs\_mom}
\end{itemize}
}

\subsection*{Obtaining and compiling Maui}

\begin{frame}
\frametitle{Maui Source Code}
Maui too is available from \texttt{www.clusterresources.com}

{\small You need to register to their site to download the code, and they \emph{may} contact you later and ask what are you going to do with their software (and offer commercial support for it)}
\mode<presentation>{\vspace*{\fill}}

\begin{center}
\includegraphics[width=0.8\textwidth]{maui-download.jpg}
\end{center}

\end{frame}

\begin{frame}
\frametitle{Building Maui}

\begin{itemize}
\item<1-> same <<\texttt{configure; make; make install}>>
\item<2-> \alert{but} there are a few issues with paths and options
	\begin{itemize}
	\item<3-> if you are linking against \texttt{libpcre} (recommended) you need to edit \texttt{include/Makefile.inc.pcre.in} so that \texttt{-lpcreposix -lpcre} are passed as two separate options (remove quotes)
	\item<4-> if \texttt{libpcre} is installed anywhere but \texttt{/usr/local} you \emph{may} need to pass some \texttt{CFLAGS=-L\ldots}
	\item<5-> if your \texttt{prefix} is anything but \texttt{/usr/local/maui} you need to set \texttt{-{}-with-spooldir} to have a consistent installation
	\end{itemize}
\end{itemize}

\end{frame}



\section{Configuration}

\subsection*{TORQUE Configuration}

\begin{frame}
\frametitle{TORQUE Common Configuration Files}

\begin{itemize}
\item<1-> \texttt{pbs\_environment} contains the environment variables for TORQUE; any minimal set will do
e.g.
\mode<article>{\par}
\hspace{5em}
\parbox{0.5\textwidth}{%
	\tt PATH=/bin:/usr/bin\\
	LANG=en\_US
}
\item<2-> \texttt{server\_name} contains the <<official>> name of the machine where \texttt{pbs\_server} runs (this is usually your master node)\\
The server name \alert{must} be identical to the FQDN\\
e.g.
\mode<article>{\par}
\hspace{5em}
\parbox{0.5\textwidth}{%
	\tt cerbero.hpc.sissa.it
}
\end{itemize}

\uncover<3->{Both these files reside in the spool directory (\texttt{/var/spool/torque})}
\end{frame}

\begin{frame}
\frametitle{TORQUE \texttt{pbs\_server} configuration}
\framesubtitle{The \texttt{nodes} file}

\texttt{server\_priv/nodes} contains the list of available computing nodes and a list of attributes for each node.
\vspace*{3ex}

{\tt%
\begin{tabular}{lll}

\uncover<2->{\small\sf\emph{node name}} & \uncover<3->{\small\sf\emph{\# of CPUs}} & \uncover<4->{%
\small\sf\em\parbox[b]{0.6\textwidth}{%
<<features>>\\
(list of arbitrary strings,\\
can be used later to select a node type)}}\\
&&\\
\uncover<2->{node01} & \uncover<3->{np=2} & \uncover<4->{opteron myri}\\
\uncover<2->{node02} & \uncover<3->{np=2} & \uncover<4->{opteron myri}\\
\uncover<2->{\ldots} && \\
\uncover<2->{node51} & \uncover<3->{np=4} & \uncover<4->{opteron IB}\\
\uncover<2->{node52} & \uncover<3->{np=4} & \uncover<4->{opteron IB}

\end{tabular}
}%tt
\end{frame}


\begin{frame}
\frametitle{TORQUE \texttt{pbs\_server} configuration}
\framesubtitle{Creating the Configuration Database}
The bulk of  \texttt{pbs\_server} configuration is written in a (binary) database. You first need to create the empty database with

\texttt{pbs\_server -t create}

This will \alert{destroy any existing configuration}, create the empty database and start a \texttt{pbs\_server}.\pause

Configuration can then be edited using the \texttt{qmgr} tool. Configuration data are written to \texttt{server\_priv/serverdb} as well as in various other files.

\mode<article>{
\begin{itemize}
\item if you are running postfix, you already have a \texttt{qmgr} somewhere in your system, so you may need to adjust some paths
\item TORQUE \texttt{qmgr} needs a running \texttt{pbs\_server} to actually write the configuration; this is because the configuration database is written by the server itself, and this in turn means that the server needs to have write permission on its own configuration
\end{itemize}
}
\end{frame}


\begin{frame}
\frametitle{TORQUE \texttt{pbs\_server} configuration}
\framesubtitle{Sample Configuration}

{\tt
\mode<presentation>{\scriptsize}
\mode<article>{\small}
\begin{minipage}{\textwidth}
\textbf{[root@borg]\# qmgr}\pause\par
Qmgr: create queue batch\par
Qmgr: set queue batch queue\_type = Execution\pause\par
Qmgr: set queue batch resources\_max.walltime = 01:00:00\par
Qmgr: set queue batch resources\_default.nodes = 1\par
Qmgr: set queue batch resources\_default.walltime = 00:01:00\pause\par
Qmgr: set queue batch enabled = True\par
Qmgr: set queue batch started = True\pause\par
Qmgr: set server managers = maui@borg.cluster\par
Qmgr: set server managers += root@borg.cluster\par
Qmgr: set server operators = maui@borg.cluster\par
Qmgr: set server operators += root@borg.cluster
\end{minipage}
}%tt

\end{frame}


\begin{frame}
\frametitle{\texttt{pbs\_mom} configuration}

\texttt{pbs\_mom} configuration can be fairly minimal, the only thing the Mom needs to know is the hostname where \texttt{pbs\_server} is running on.

\uncover<2->{Useful additions include log configuration, how to handle user file copy and which filesystem to monitor for available space.}
\mode<presentation>{\vspace*{\fill}}

\texttt{mom\_priv/config}:
\vspace{1ex}

{\tt
\begin{tabular}{ll}
\uncover<1->{\$clienthost	& master.hpc}\\
\uncover<2->{\$logevent	& 0x7f}\\
\uncover<2->{\$usecp		& *:/home /home}
\end{tabular}
\begin{tabular}{l}
\uncover<2->{size[fs=/local\_scratch]}
\end{tabular}
}%tt


\end{frame}


\subsection*{Maui Configuration}

\begin{frame}
\frametitle{Maui Configuration}
\framesubtitle{How to Connect to Resource Manager}
\begin{itemize}
\item simpler approach: a single configuration file (\alert{\texttt{maui.cfg}})
\item<2-> Maui needs to know what RM to connect to and how
\end{itemize}

\uncover<2->{{\tt\small
\begin{center}
\begin{tabular}{ll}
SERVERHOST	&	borg.cluster\\
RMCFG[BORG.CLUSTER] &	TYPE=PBS\\
RMPOLLINTERVAL	&	00:00:30\\
SERVERPORT	&	42559\\
SERVERMODE	&	NORMAL\\
ADMIN1		&	root
\end{tabular}
\end{center}
}}%tt,small

\mode<article>{\ar{\texttt{SERVERHOST} is the same we defined for TORQUE.}}

\mode<article>{\ar{User(s) listed as \texttt{ADMIN1} have full control over Maui. The first user in the list must be used to run Maui itself; if you want to run Maui as a non-privileged user, put this username here. The user Maui is running as needs to be able to control \texttt{pbs\_server}.}}
\end{frame}


\begin{frame}
\frametitle{Maui Configuration}
\framesubtitle{Job Prioritization}

Job priority is recomputed at each scheduler iteration, according to site-defined parameters. If no parameters are set only queue time is taken into account, i.e. the scheduling is strictly FIFO.
\mode<presentation>{\vspace*{\fill}}

\uncover<2->{Priority components include:}
\begin{itemize}
\item<2-> \alert{\textbf{Queue Time}}: how long the job has been \alert{idle} in the queue
\item<3-> \alert{\textbf{Credentials}}: a static priority can be assigned on a user, group, queue basis
\item<4-> \alert{\textbf{Fair Share}}: historical usage data
\item<5-> \alert{\textbf{Resources}} requested for the job
\end{itemize}

\end{frame}


\begin{frame}
\frametitle{Maui Configuration}
\framesubtitle{Job Prioritization: Queue Time and Credentials}

{\tt
\begin{tabular}{ll}
QUEUETIMEWEIGHT & 1\\\pause
XFACTORWEIGHT  & 10\\\pause
CLASSCFG[batch] & PRIORITY=1\\
CLASSCFG[fast] & PRIORITY=1000\\\pause
GROUPCFG[guests] & PRIORITY=1\\
GROUPCFG[users] & PRIORITY=1000\\
GROUPCFG[devel] & PRIORITY=10000\\\pause
USERCFG[DEFAULT] & PRIORITY=2000\\
USERCFG[luser1] & PRIORITY=0
\end{tabular}
}%tt

\mode<article>{
\begin{itemize}
\item a high \texttt{QUEUETIMEWEIGHT} makes the scheduling <<more FIFO>>
\item $ \textit{XFactor} = \frac{\textit{QueueTime} + \textit{WallClockLimit}}{\textit{WallClockLimit}} $ so a high \texttt{XFACTORWEIGHT} favors <<short>> jobs: this usually makes users happy in the short term, but can degrade overall cluster performance
\end{itemize}
}

\end{frame}


\begin{frame}
\frametitle{Maui Configuration}
\framesubtitle{Job Prioritization: Fair Share}

The FS priority component must be explicitly enabled by setting its weight to a non-sero value.
\mode<presentation>{\vspace*{\fill}}

{\tt
\begin{tabular}{llp{0.5\textwidth}}
FSINTERVAL	&	86400	& {\sf\em\scriptsize duration of each FS window}\\
FSDEPTH		&	30	& {\sf\em\scriptsize number of FS windows}\\
FSDECAY		&	0.90	& {\sf\em\scriptsize decay factor applied to older FS windows}\\\pause
FSWEIGHT	&	1\\
FSGROUPWEIGHT	&	240\\
FSUSERWEIGHT	&	10\\
\end{tabular}
}%tt

\mode<article>{}

\end{frame}

\begin{frame}
\frametitle{Maui Configuration}
\framesubtitle{Job Prioritization: Fair Share}

Usage targets can be set on a per-user, per-group and per-queue basis.
\mode<presentation>{\vspace*{\fill}}

{\tt
\begin{tabular}{ll}
USERCFG[DEFAULT] & FSTARGET=1\\
GROUPCFG[users] & FSTARGET=30\\
GROUPCFG[devel] & FSTARGET=40\\
\end{tabular}
}%tt
\mode<presentation>{\vspace*{\fill}}\pause

You can set also FS floors or caps so that priority is affected only when usage drops below the floor or goes above the cap:
\mode<presentation>{\vspace*{\fill}}

{\tt
\begin{tabular}{llp{0.3\textwidth}}
GROUPCFG[guests] & FSTARGET=5- & {\sf\em\scriptsize give a negative priority component if usage is above 5\%}\\
USERCFG[master] & FSTARGET=20+ & {\sf\em\scriptsize give a priority boost if \mbox{usage} is below 20\%}
\end{tabular}
}%tt

\end{frame}

%\section{Advanced Configuration Examples}

\subsection*{Prologue and Epilogue}

\begin{frame}
\frametitle{Prologue \& Epilogue scripts}

\texttt{pbs\_mom} looks for scripts in its configuration directory \texttt{mom\_priv}. If found, the \alert{\texttt{prologue}} script is executed just before job start and the \alert{\texttt{epilogue}} script at job termination.

The \texttt{prologue} script performs any initialization that is requered on the node for the job to run, while the \texttt{epilogue} undoes the modifications.
\pause

\mode<presentation>{\vspace*{\fill}}

\begin{center}
\textbf{\texttt{/etc/security/access.conf}}
\end{center}

\begin{tabular}{p{0.4\textwidth}cp{0.4\textwidth}}
\emph{\small before \texttt{prologue}} &&  \emph{\small after \texttt{prologue}}\\
\\
\texttt{-:ALL EXCEPT root:ALL} & $\longrightarrow$ & \texttt{-:ALL EXCEPT root someuser:ALL} \\
\emph{\small disallows login to everybody except root, from anywhere} && \emph{\small now allows \texttt{someuser} to \mbox{login}}

\end{tabular}

\end{frame}



\section{Diagnostics \& Troubleshooting}

\subsection*{TORQUE Diagnostics}

%\begin{frame}
%\frametitle{\texttt{tracejob}}
%boh...
%
%\end{frame}

\begin{frame}
\frametitle{\texttt{momctl}}

Query and control remote \texttt{pbs\_mom}:\pause

\textbf{\texttt{\# momctl -d3 -h i602}}\pause

{\tt\scriptsize
Host: i602/i602.hpc   Server: master.hpc   Version: 1.2.0p6

\begin{tabular}{ll}
HomeDirectory:         &/var/spool/PBS/mom\_priv\\
MOM active:            &6907718 seconds\\
Last Msg From Server:  &213582 seconds (DeleteJob)\\
Last Msg To Server:    &1 seconds\\
Server Update Interval:&45 seconds\\
Init Msgs Received:    &10 hellos/2 cluster-addrs\\
Init Msgs Sent:        &190 hellos\\
LOGLEVEL:              &0 (use SIGUSR1/SIGUSR2 to adjust)\\
Communication Model:   &RPP\\
TCP Timeout:           &20 seconds\\
Prolog Alarm Time:     &300 seconds\\
Alarm Time:            &0 of 10 seconds\\
Trusted Client List:   & \ldots\\
JobList:               &NONE
\end{tabular}

diagnostics complete
}%tt

\end{frame}


\subsection*{Maui Diagnostics}

%diagnose -p

%checkjob

\begin{frame}
\frametitle{\texttt{checknode}}

Check who is doing what on a node and show node capabilities

\textbf{\texttt{\# checknode a034}}\pause

{\tt\scriptsize
checking node a034

State:      Busy  (in current state for 1:13:38:12)

Configured Resources: PROCS: 2  MEM: 3949M  SWAP: 7242M  DISK: 59G

Utilized   Resources: PROCS: 2  DISK: 10G

Dedicated  Resources: PROCS: 2

Opsys:       DEFAULT  Arch:      [NONE]

Speed:      1.00  Load:       2.000 (ProcSpeed: 2600)

Network:    [DEFAULT]

Features:   [myri][opteron][opteron-sc]\ldots

Attributes: [Batch]

Classes:    [smp2 2:2][smp4 2:2][mpi4 0:2][mpi8 2:2]\ldots

Total Time: 25:14:33:36  Active: 25:04:53:26 (98.43\%)

Reservations:

  Job '30069'(x2)  -1:13:38:44 -> 2:10:20:16 (3:23:59:00)

JobList:  30069
}%tt
\end{frame}

%showbf



%\subsection*{Reservations}

%setres

%showres
\mode<presentation>{
\begin{frame}
\begin{center}
\includegraphics[width=0.7\textwidth]{thatsall.png}
\end{center}

\vspace{\fill}
\begin{flushright}
\small
\url{<calucci@sissa.it>}
\end{flushright}

\end{frame}
}

\end{document}
