\documentclass[twoside]{article}[12pt]

% include packages; if you don't like the big default margins, 
% then set the geometry BEFORE including the votutorial-style
\usepackage[a4paper, top=2cm, bottom=2cm, left=2cm, right=2cm]{geometry}

\usepackage[gavo]{votutorial}

% for syntax highlighting:
%\usepackage{listings}

% more tool-commands:
\newcommand{\CosmoSim}{%
\link{CosmoSim}{http://www.cosmosim.org/}%
}
\newcommand{\uwsclient}{%
\link{uws-client}{http://github.com/aipescience/uws-client}%
}
\newcommand{\httpie}{%
\link{httpie}{http://httpie.org/}%
}

\newcommand{\code}[1]{%
\texttt{#1}
}


\lstset{style=daiquiri, language=bash}


\title{Introduction to UWS}
\author{Kristin Riebe}
%\titlepicture{<Your Picture>}

%\def\runtitle{RAVE with TOPCAT and Aladin}
%\def\abstractname{Introduction}
%\fancyhead[LE,RO]{\runtitle}

\begin{document}
\maketitle

\begin{abstract}
This tutorial shows how to use the UWS interface of web services with job queues to access their service/data from the command line.
We will use a bash shell with httpie as well as a Python uws-client to demonstrate the basic functionality. 
This could be used for a number of services, e.g. 
CosmoSim, RAVE database, APPLAUSE plate archive, but also for VO TAP services like Millennium TAP, GAVO Datacenter (DaCHS), the Canadian Data Center and others. 
%The uws-client can also be used to check if a UWS-service is up and running and responding adequately.
\end{abstract}

\usedsoftware{\uwsclient, \httpie, \CosmoSim}

\section{Introduction}
UWS (Universal Worker Service) is an IVOA standard for asynchronous web services with job management. That means users do not get their results immediately after a request, but they send jobs to the web service. 
The service puts these jobs into a queue and they are processed consecutively. The job results are then stored on the server and can be retrieved by the user at a later time. 
This tutorial shows how to interact with such an asynchronous UWS service from the command line.

\subsection{Installing software}
For this tutorial, we need a command line tool that can send GET and POST requests to a URL. We suggest to use \httpie\ or our \uwsclient\ and will show each step with both tools. They are written in Python, so all you need is a Python installation (2.7 upwards). Using the \code{pip} package manager, they should be easy to install:

\begin{itemize}
\item \httpie:
\begin{lstlisting}[language=bash]
pip install httpie
\end{lstlisting}

\item \uwsclient:\\
Download the uws-client from \url{http://github.com/aipescience/uws-client/} and install using \code{pip}:
\begin{lstlisting}[language=bash]
git clone https://github.com/aipescience/uws-client
cd uws-client
pip install .
\end{lstlisting}
%
In principle, this should install the 
necessary packages and make an alias \code{uws} to \mbox{\code{uws-client/cli/main.py}.} Look into the README for more information. 
\end{itemize}

\section{Basic UWS steps with CosmoSim using httpie}
The following steps are shown using \httpie, the equivalent steps using \uwsclient\ are given in section \ref{sec:uwsclient}.
We will use \CosmoSim\ as an example UWS service for this tutorial. It provides access to data from cosmological simulations, like dark matter halo catalogs or merger trees, stored in a database.
%
For using this UWS service, one needs a username and password. You can use the following credentials\footnote{The password will change from time to time. Please contact us for the current demo password or register to get your own personal account.} for this tutorial:\\
\menu{username}: \entry{cosmodemo}, \menu{password}: \entry{gavo}.

\subsection{Get the job list}
For querying a list of all jobs for a user, we need to send a GET request to the base UWS URL:\\
\url{https://www.cosmosim.org/uws/query}.

    \begin{lstlisting}
http --auth cosmodemo:gavo https://www.cosmosim.org/uws/query
    \end{lstlisting}


\subsection{Create a job}
Send a POST request with job parameters to create a job. You can get the required job parameters from the documentation for each web service.
    \begin{lstlisting}
http --auth cosmodemo:gavo --form --follow POST \  
       https://www.cosmosim.org/uws/query \
       query="SELECT x,y,z FROM MDR1.FOF LIMIT 10"
    \end{lstlisting}
%
This returns the job details of the created job, including a new (and unique) jobid. The job is now in PENDING phase.
It is acccepted by the server, but still needs another POST request to really start the job.

\subsection{Start a job}
To start a previously created job, send a POST request to its phase with parameter \code{PHASE=RUN}.

    \begin{lstlisting}
http --auth cosmodemo:gavo --form --follow POST \  
       https://www.cosmosim.org/uws/query/<jobid>/phase PHASE=RUN
    \end{lstlisting}
If you look for the job's phase now, the job should be QUEUED, EXECUTING or already COMPLETED.


\subsection{Get job details/phase}
You can check the job's details and especially its status by sending a GET request to the job's url.
Replace \code{jobid} in the following commands by the jobid returned from the previous steps.
    \begin{lstlisting}
http --auth cosmodemo:gavo GET https://www.cosmosim.org/uws/query/<jobid>
http --auth cosmodemo:gavo GET https://www.cosmosim.org/uws/query/<jobid>/phase
    \end{lstlisting}


\subsection{Retrieve results}
After the job is completed, you can ask a UWS service where the results are stored and retrieve them.

    \begin{lstlisting}
http --auth cosmodemo:gavo GET https://www.cosmosim.org/uws/query/<jobid>/results
    \end{lstlisting}
    Pick one of the results and download:
    \begin{lstlisting}
http --auth cosmodemo:gavo --download GET <resulturl>
    \end{lstlisting}


\subsection{Delete job}
Jobs that are not needed anymore should be removed. This is done by sending a DELETE request to the job.
    \begin{lstlisting}
http --auth cosmodemo:gavo --follow DELETE https://www.cosmosim.org/uws/query/<jobid>
    \end{lstlisting}





\section{Steps using the uws-client}
\label{sec:uwsclient}

The \uwsclient\ is more convenient to use than httpie, because it already formats the results nicely for you. 
It also offers some more features like job list filtering by status (phase) or creating and starting a job in one step.

\subsection{Get the job list}
    \begin{lstlisting}
uws --user cosmodemo --password gavo \
    --host https://www.cosmosim.org/uws/query \
    list  
    \end{lstlisting}  

    The uws-client also allows you to filter jobs by their status, called \emph{phase}. The main phases are PENDING, QUEUED, EXECUTING, COMPLETED, ERROR and ABORTED. For getting all jobs with ERROR-phase, use:
    \code{uws  [...]  list --error}


\subsection{Create a job}
    \begin{lstlisting}
uws --user cosmodemo --password gavo \
    --host https://www.cosmosim.org/uws/query \
    job new
    query="SELECT x,y,z FROM MDR1.FOF LIMIT 10"
    \end{lstlisting}
%
This returns the job details of the created job, including a new (and unique) jobid. For immediately starting the job already, append \code{--run} to the command above.

\subsection{Start a job}
    \begin{lstlisting}
uws --user cosmodemo --password gavo \
    --host https://www.cosmosim.org/uws/query \
    job new
    query="SELECT x,y,z FROM MDR1.FOF LIMIT 10"
    \end{lstlisting}


\subsection{Get job details}
Replace \code{jobid} in the following commands by the jobid returned from the previous steps.
    \begin{lstlisting}
uws --user cosmodemo --password gavo \
    --host https://www.cosmosim.org/uws/query \
    job show <jobid>
    \end{lstlisting}


\subsection{Retrieve results}
    \begin{lstlisting}
uws --user cosmodemo --password gavo \
    --host https://www.cosmosim.org/uws/query \
    job results <jobid> votable.xml
    \end{lstlisting}
    This already downloads the specified result. Use \code{job show} to see the result-ids and urls.


\subsection{Delete job}
    \begin{lstlisting}
uws --user cosmodemo --password gavo \
    --host https://www.cosmosim.org/uws/query \
    job delete <jobid>
    \end{lstlisting}
%This removes the job results, but the metadata of this job may still be kept

\end{document}
