\documentstyle[11pt,psfig]{article}

\topmargin=0in
\oddsidemargin=0in
\textheight=9in
\textwidth=6.5in

\setlength{\parindent}{0cm}
\setlength{\parskip}{1.5ex}
\pagestyle{myheadings}
\begin{document}


{\LARGE
\centerline{\bf A Tutorial on the IQL Query Language}
\centerline{AutoMed Technical Report No. 28, Version 1.0}
\centerline{Alex Poulovassilis, February 2004}
}

\section{Introduction}

IQL is a typed, functional language. 
It supports strings (e.g. 'Computer Science'), 
booleans (True, False), real numbers (e.g. 10.51) and integers. 
It also supports tuples (e.g. {\tt \{1,2,3\}}) and lists (e.g. {\tt [1,2,3]}).
The tuple and list constructors can be arbitrarily nested,
e.g. {\tt [\{[1,2,3],[4,5]\}, \{[3,4],[1,2,5]\}, \{[],[6,7,8]\}]}
is a list of pairs, each of whose components is a list of numbers. 

A number of built-in functions are provided
e.g. {\tt (+)}, {\tt (-)}, {\tt (*)}, {\tt (/)}, {\tt (=)}, {\tt (!=)},
($<$), ($>$), ($<$=), ($>$=), {\tt and}, {\tt or}, {\tt not}, {\tt if}. 
Currently most built-in functions are only supported in prefix form, the
exceptions being the binary operators {\tt ++} and {\tt --} (see below) which can
be used both prefix (as {\tt (++)} and {\tt (--)}, respectively) 
and infix (as {\tt ++} and {\tt --}). 

New, anonymous, functions can be defined by using {\bf lambda abstractions}. For example:
\begin{verbatim}
lambda  {x,y,z} ((*) ((+) x y) z)
\end{verbatim}
defines a function which takes a triple, adds the first two components and
multiplies by the third one.  So
\begin{verbatim}
(lambda  {x,y,z} ((*) ((+) x y) z)) {3,4,5}
\end{verbatim}
returns {\tt (3+4)*5} = {\tt 35}

Also supported are `let' expressions, of the form {\tt let v = q1 in q2},
for any queries {\tt q1} and {\tt q2}. These
translate internally into an expression of the form
\begin{verbatim}
(lambda v q2) q1
\end{verbatim}
For example, the query
\begin{verbatim}
let v = ((+) 200 500) in ((*) v v)
\end{verbatim}
returns {\tt 700 * 700} = {\tt 490000}, the query
\begin{verbatim}
let f = (lambda  {x,y,z} ((*) ((+) x y) z)) in ((+) (f {1,2,3}) (f {3,4,5})) 
\end{verbatim}
returns {\tt ((1+2)*3) + ((3+4)*5)} = {\tt 44}. 

All IQL queries are expressed internally in the form of Abstract Syntax Graphs
(ASGs) --- for details, see the AutoMed technical report by Jasper,
Poulovassilis, Zamboulis on ``Processing IQL Queries and Migrating Data in the AutoMed Toolkit''.

IQL uses the {\bf list} as its collection type.  If the ordering of elements within
collections is ignored, then its operators are faithful to the expected bag
semantics.  To remove duplicates from collections, an operator {\tt distinct}
is provided.  We give some examples of IQL queries below, which illustrate the use of
most of its features.

\section{Expressing the Relational Algebra in IQL}

Consider two collections R and S both consisting of 3-tuples of
numeric values.  We can compute the {\bf union} of R and S by:
\begin{verbatim}
R ++ S
\end{verbatim}
Note that this does not eliminate duplicates. To do so, we can use
the {\tt distinct} operator:
\begin{verbatim}
distinct (R ++ S)
\end{verbatim}
or, equivalently, the {\tt setUnion} operator:
\begin{verbatim}
setUnion R S
\end{verbatim}


We can compute the {\bf difference} of R and S by:
\begin{verbatim}
R -- S
\end{verbatim}
For each member r of R, this operator returns 
max(0,occurs(r,R) -- occurs(r,S)) instances of r in the result (where occurs(r,R) denotes 
the number of instances of r in R).  Note that this reduces to just ordinary set
difference if R and S have no duplicate elements.  To return members of R
which do not appear in S, we use:
\begin{verbatim}
[r | r <- R; not (member S r)]
\end{verbatim}


We can compute the {\bf intersection} of R and S by:
\begin{verbatim}
intersect R S
\end{verbatim}
For any value v, this operator returns min(occurs(v,R),occurs(v,S))
instances of v in the result.  Note this reduces to just ordinary
set intersection if R and S have no duplicate elements.  To return
members of R which also appear in S, we use:
\begin{verbatim}
[r | r <- R; member S r]
\end{verbatim}

We can use {\em comprehension syntax} to express 
{\bf projection}\footnote{The general syntax of comprehensions is
$$
[e\ |\ Q_1;\ \ldots;\ Q_n]
$$
Here, $e$ is an expression, termed the {\em head} of the comprehension,
and $Q_1$ to $Q_n$ are {\em qualifiers}, where $n \geq 0$.
Each qualifier is either a {\em filter} or a {\em generator}.
Generators have syntax $p \leftarrow s$,
where $p$ is a pattern and $s$ is a collection-valued expression.  
A {\em pattern} is an expression consisting of constructors and variables 
only (no functions or lambda abstractions).  
The variables of $p$ are successively bound by iterating through $s$. Any
variables appearing in the head $e$ inherit these bindings. A filter is a
boolean-valued expression, which must be satisfied by the values generated by
the generators in order for these values to contribute to the final result of
the comprehension.  Comprehensions are a convenient syntax and add no extra 
expressiveness to languages such as IQL since they
translate into applications of the $flatmap$ and $if$ functions:
\newline
\newline
\begin{math}
\begin{array}{llllllll}
& ~~~~ [e\ |\ p \leftarrow s;\, Q]  & \equiv & flatmap\  (lambda\ p\ [e\ |\ Q])\  s \\
& ~~~~ [e\ |\ e';\, Q]              & \equiv & if\ e'\ [e\ |\ Q]\ [\, ] \\
& ~~~~ [e\ |\ ]                     & \equiv & [e]
\end{array}
\end{math}
\newline
}.  For example, we can project the first and third
components of tuples in R by:
\begin{verbatim}
[{x,z} | {x,y,z} <- R]
\end{verbatim}
This query can be read as stating:
\begin{verbatim}
for every 3-tuple {x,y,z} in R do
    return {x,z}
\end{verbatim}

We can also use comprehension syntax to express {\bf selection}.
For example, to return tuples of R whose second component is greater
than 10:
\begin{verbatim}
[{x,y,z} | {x,y,z} <- R; (>) z 10]
\end{verbatim}
This query can be read as stating:
\begin{verbatim}
for every 3-tuple {x,y,z} in R do 
    if z > 10 
    then return {x,z}
\end{verbatim}

We can also use comprehension syntax to express {\bf cartesian product}.
For example to return the cartesian product of R, S and a third
relation T consisting of 1-tuples:
\begin{verbatim}
[{x1,y1,z1,x2,y2,z2,x3} | {x1,y1,z1} <- R; {x2,y2,z2} <- S; {x3} <- T]
\end{verbatim}
This query can be read as stating:
\begin{verbatim}
for every 3-tuple {x1,y1,z1} in R do
    for every 3-tuple {x2,y2,z2} in S do
        for every 1-tuple {x3} in T do
            return {x1,y1,z1,x2,y2,z2,x3}
\end{verbatim}


Finally, we can use comprehension syntax to express {\bf joins}.
For example, to join R and S over their second column
and return a set of 6-tuples:
\begin{verbatim}
[{x1,y1,z1,x2,y2,z2} | {x1,y1,z1} <- R; {x2,y2,z2} <- S; (=) y1 y2]
\end{verbatim}
This query can be read as stating:
\begin{verbatim}
for every 3-tuple {x1,y1,z1} in R do
    for every 3-tuple {x2,y2,z2} in S do
        if y1 = y2
        then return {x1,y1,z1,x2,y2,z2}
\end{verbatim}

To eliminate the duplicated join column from the 
above result and return a set of 5-tuples:
\begin{verbatim}
[{x1,y1,z1,x2,z2} | {x1,y1,z1} <- R; {x2,y2,z2} <- S; (=) y1 y2]
\end{verbatim}
to join R and S over their second and third columns, eliminating
the duplicated join columns from the result:
\begin{verbatim}
[{x1,y1,z1,x2} | {x1,y1,z1} <- R; {x2,y2,z2} <- S; (=) y1 y2; (=) z1 z2]
\end{verbatim}
to perform a theta-join of R and S such that the first component
of tuples of R is less than the third component of tuples of S:
\begin{verbatim}
[{x1,y1,z1,x2,y2,z2} | {x1,y1,z1} <- R; {x2,y2,z2} <- S; (<) x1 z2]
\end{verbatim}
and to perform a more complex theta-join of R and S, illustrating
the use of the {\tt and}, {\tt or} and {\tt not} operators:
\begin{verbatim}
[{x1,y1,z1,x2,y2,z2} | {x1,y1,z1} <- R; {x2,y2,z2} <- S; 
                       not (and (or ((<) x1 z2) ((>) x1 z2)) ((=) y1 y2))]
\end{verbatim}

It is important to note that IQL queries can be arbitrarily {\bf nested}, so
the collections resulting from any of the above operations can form the
argument of another operator.  For example, suppose that R and S both have
scheme (Name,Gender,Age). The IQL query corresponding to the following
relational algebra expression:
$$
\sigma_{Age > 25}(\pi_{Name,Age}((\sigma_{Gender='Male'}R)\ - \ (\sigma_{Gender='Male'}S)))
$$
is:
\begin{verbatim}
[{n,a} | {n,a}<-[{n,a} | {n,g,a} <- [{n,g,a}|{n,g,a}<-R; (=) g 'Male'] -- 
                                    [{n,g,a}|{n,g,a}<-S; (=) g 'Male']
                ]; 
         (>) a 25]
\end{verbatim}
We can simplify this by merging the outermost selection and projection
operations:
\begin{verbatim}
[{n,a} | {n,g,a} <- [{n,g,a}|{n,g,a}<-R; (=) g 'Male'] -- 
                    [{n,g,a}|{n,g,a}<-S; (=) g 'Male']; 
         (>) a 25]
\end{verbatim}
This optimisation is known as {\bf loop fusion}. 

{\bf Exercise}

Given that R and S both have scheme (Name,Gender,Age), construct an
IQL query corresponding to the following relational algebra expression:
$$
\sigma_{Name='Jones'\ AND\ Gender='Female'}
(\pi_{Name,Gender}((\sigma_{Age > 50}R)\ \cap\ (\sigma_{Age < 50}S)))
$$
{\em Hint}: start from the innermost sub-queries and construct the IQL
query outwards. 

%
%Answer:
%[{n,g} | {n,g} <- [{n,g} | {n,g,a} <- intersect [{n,g,a}|{n,g,a}<-R; (>) a 50]
%                                                [{n,g,a}|{n,g,a}<-S; (<) a 50]
%                  ];
%         (=) n 'Jones';
%         (=) g 'Female']
%
% Or, simplifying the iteration over {n,g} (loop fusion)
%
%[{n,g} | {n,g,a} <- intersect [{n,g,a}|{n,g,a}<-R; (>) a 50]
%                              [{n,g,a}|{n,g,a}<-S; (<) a 50];
%         (=) n 'Jones';
%         (=) g 'Female']
%

\section{Grouping and Aggregation Operations in IQL}

IQL supports the expected grouping and aggregation operations.
For example, to count the number of elements in R:
\begin{verbatim}
count R
\end{verbatim}
to sort R:
\begin{verbatim}
sort R 
\end{verbatim}
to remove duplicates from R:
\begin{verbatim}
distinct R 
\end{verbatim}

To return the maximum value in R's second column:
\begin{verbatim}
max [y | {x,y,z} <- R]
\end{verbatim}
and the minimum:
\begin{verbatim}
min [y | {x,y,z} <- R]
\end{verbatim}
To return the sum over R's third column:
\begin{verbatim}
sum [z | {x,y,z} <- R]
\end{verbatim}
and the average:
\begin{verbatim}
avg [z | {x,y,z} <- R]
\end{verbatim}


To group R on its first column:
\begin{verbatim}
group [{x,{y,z}} | {x,y,z} <- R]
\end{verbatim}
on its second column:
\begin{verbatim}
group [{y,{x,z}} | {x,y,z} <- R]
\end{verbatim}
and on its second and third columns:
\begin{verbatim}
group [{{y,z},x} | {x,y,z} <- R]
\end{verbatim}

Generally, we observe that {\tt group} expects as an argument a collection of
pairs, and groups them on their first component.  If the result needs to be
reordered, then this can be done using a comprehension.  For example, to
group R on its second and third column, and present the results with the
original ordering of the tuple components:
\begin{verbatim}
[{x,y,z} | {y,z,x} <- group [{{y,z},x} | {x,y,z} <- R]]
\end{verbatim}


To group a collection and apply an aggregation function to each
group, we can use the operator {\tt gc}.  In general, 
{\tt gc agFun xs} groups a collection of pairs {\tt xs} on their 
first component, and then applies the aggregation function 
{\tt agFun} to the second component.  For example, to group
R on its first and second columns and return the maximum of the
values in the third column for each group:
\begin{verbatim}
gc max [{{x,y},z} | {x,y,z} <- R]
\end{verbatim}
to group R on its second and third column and return the total value 
of the values in the first column for each group:
\begin{verbatim}
gc sum [{{y,z},x} | {x,y,z} <- R]
\end{verbatim}
to group R on its first column and return the minimum of the
values in the third column for each group:
\begin{verbatim}
gc min [{x,z} | {x,y,z} <- R]
\end{verbatim}
to group R on its third column and return the average of the
values in the first column for each group:
\begin{verbatim}
gc avg [{z,x} | {x,y,z} <- R]
\end{verbatim}

{\tt gc} is an example of a higher-order operator
(an operator is `higher-order' if it takes another operator as
an argument).  IQL supports two other higher-order operators:
{\tt map} and {\tt flatmap}. 

{\tt map f xs} applies a function {\tt f} to each member of a 
collection {\tt xs}. For example, to add 10 to the first component
of each tuple of R:
\begin{verbatim}
map (lambda {x,y,z} {(+) 10 x, y, z}) R
\end{verbatim}
We can always use comprehension syntax instead of using {\tt map} e.g.
\begin{verbatim}
[{(+) 10 x, y, z} | {x,y,z} <- R]
\end{verbatim}
More generally, {\tt map f xs} is equivalent to {\tt [f x | x <- xs]}

{\tt flatmap f xs} applies a collection-valued function {\tt f} 
to each member of a collection {\tt xs} and appends the resulting
collections.  For example:
\begin{verbatim}
flatmap (lambda {x,y,z} [{(+) 10 x, y, z}]) R
\end{verbatim}
gives the same result as the previous two queries. 


\section{Using IQL in AutoMed}

Currently IQL is used in AutoMed in two ways: to express the queries within
{\sf add}, {\sf delete}, {\sf extend} and {\sf contract} transformations, and
to express queries on schemas defined within AutoMed's Schemas and
Transformations Repository (STR).  Such schemas may be data source schemas,
intermediate schemas or global schemas.  In all cases, the individual
constructs of a schema are identified by their {\em scheme} within IQL
queries, and they return lists of values.

For example, consider a schema construct {\tt <<Student,name>>} representing
the attribute {\tt name} of a table {\tt Student}, which has a
single-attribute primary key {\tt studentId}, say.  The following IQL query
counts the number of distinct students' names and sees if this equals the
number of student id values:
\begin{verbatim}
(=) (count (distinct [n | {s,n} <- <<Student,name>>])) (count <<Student>>)   
\end{verbatim}
Note that this is how a primary key constraint can be defined in IQL.

As another example, suppose we have in a global relational schema attributes
{\tt <<Student,address>>} and {\tt <<Student,age>>} possibly sourced from
different databases, and we want to return the set of all known students with
all their known address and age information.  Here is a query that does this:
\begin{verbatim}
[{s,a,g} | {s}<-<<Student>>;
           {s1,a}<-<<Student,address>>; (=) s s1;
           {s2,g}<-<<Student,age>>; (=) s s2]
++
[{s,UnknownAddress,g} | {s}<-<<Student>>;
           {s1,g}<-<<Student,age>>; (=) s s1;
           not (member [s | {s,a} <- <<Student,address>>] s)]
++
[{s,a,UnknownAge} | {s}<-<<Student>>;
           {s1,a}<-<<Student,address>>; (=) s s1;
           not (member [s | {s,g} <- <<Student,age>>] s)]
++
[{s,UnknownAddress,UnknownAge} | {s}<-<<Student>>;
           not (member [s | {s,a} <- <<Student,address>>] s);
           not (member [s | {s,g} <- <<Student,age>>] s)]
\end{verbatim}

We observe that this query is expressing an outerjoin of {\tt <<Student>>},
{\tt <<Student,address>>} and {\tt <<Student,age>>} over their common 
{\tt studentId} component. 

User-specified identifiers such as {\tt UnknownAge} and {\tt UnknownAddress}
above are treated as constants by the IQL evaluator, i.e. they are returned
as-is within query results.  However, at present there is no type-checking in
IQL (either static or dynamic), and so queries such as {\tt (+) 1 UnknownAge}
will generate a run-time exception.  It is currently up to the programmer to
guard against such type-incorrect expressions being passed to the IQL
evaluator for evaluation.

\appendix

\section{IQL Syntax}
\label{IQLSyntax}

\begin{ttfamily}
\begin{tabbing}
\hspace*{3.5cm}\=\hspace{3mm}\= \kill
query ::=\>\>expr\\
\>|\>Let VarToken Equal query In query\\
\>|\>query Append query\\
\>|\>query Difference query\\
\>|\>query expr\\
\\
expr ::=\>\>NumToken\\
\>|\>StrToken\\
\>|\> VarToken\\
\>|\> Any\\
\>          |\> BoolToken\\
\>          |\> Void\\
\>          |\> scheme\\
\>          |\> VarToken Colon scheme\\
\>          |\> LSB query Bar quals RSB\\
\>          |\> LSB RSB\\
\>          |\> LSB seq RSB\\
\>          |\> LCB seq RCB\\
\>          |\> LRB query RRB\\
\>          |\> Lambda pattern expr\\
\\
seq ::=\>\> 	seq Comma query\\
\>          |\> query\\
\\
quals ::=\>\>   qual SemiColon quals\\
\>          |\> qual\\
\\
qual ::=\>\>    query\\
\>          |\> pattern LArrow query\\
\\
pattern ::=\>\> VarToken\\
\>          |\> LCB pattern\_seq RCB\\
\\
pattern\_seq ::=\>\> pattern\_seq Comma pattern\\
\>          |\> pattern\\
\\
scheme ::=\>\> LDAB scheme\_seq RDAB\\
\\
scheme\_seq ::=\>\>   scheme\_element\\
\>               |\> scheme\_seq Comma scheme\_element\\
\\
scheme\_element ::=\>\>  VarToken\\
\>                   |\> StrToken\\
\>                   |\> NumToken\\
\>                   |\> scheme\\
\\
BoolToken =  \>\>"True" | "False"\\
\\
StrToken =  \>\>$\backslash$'[ \^\ $\backslash$']*$\backslash$'\\
\\
NumToken  = \>\>[0-9]+|([0-9]+)(".")([0-9]+)\\
\\
VarToken = \>\>"(+)" | "(-)" | "(*)" | "(/)" |  \\
\>|\> "(=)" | "(!=)" | "(<)" | "(>)" | "(<=)" | "(>=)" \\
\>|\> [A-Za-z\_][A-Za-z0-9\_\$]*\\
\\
Let = "let"\\
In = "in"\\
Equal = "="   \\
Append = "++"\\
Difference = "--"\\
SemiColon = ";"  \\
LArrow = "<-" \\
Comma = "," \\
Bar = "|"   \\
LSB = "["   \\
RSB = "]"   \\
LRB = "("  \\
RRB = ")"   \\
LCB = "\{"  \\
RCB = "\}"   \\
Any = "Any" \\
Lambda = "lambda" \\
LDAB = "<<" \\
RDAB = ">>" \\
Colon = ":"  \\
Void = "Void" \\
\end{tabbing}
\end{ttfamily}

\section{Built-In Functions}
\label{Functions}

\subsection{Unary Operators}

\begin{verbatim}
not      /* (not e) returns the negation of e */ 
count    /* (count xs) returns the length of the list xs */ 
sort     /* (sort xs) sorts the list xs */ 
distinct /* (distinct xs) removes duplicates from the list xs */ 
group    /* (group xs) groups a list of pairs xs on their first component */ 
         /* and returns a list of pairs whose second components are lists */ 
max      /* (max xs) returns the maximum of a list of numbers xs */
min      /* (min xs) returns the minimum of xs */
sum      /* (sum xs) returns the sum of xs  */
avg      /* (avg xs) returns the average of xs */
\end{verbatim}

\subsection{Binary Infix Operators}

\begin{verbatim}
++       /* (xs ++ ys) appends the two lists xs and ys */ 
--       /* (xs -- ys) returns the monus (bag difference) of xs and ys */ 
\end{verbatim}

\subsection{Binary Prefix Operators}

\begin{verbatim}
(+) (-) (*) (/)
(=) (!=) (>) (<) (>=) (<=)
and or 

setUnion  /* (setUnion xs ys) appends xs and ys and then removes duplicates */
sub       /* (sub xs ys) returns whether xs is a sub-bag of ys */
intersect /* (intersect xs ys) returns the bag intersection of xs and ys */
member    /* (member xs x) returns whether x is a member of xs */ 

map       /* (map f xs) applies a function f to each member of xs */ 
flatmap   /* (flatmap f xs) applies a list-valued function f to each member */ 
          /* of xs and then appends the resulting lists */ 
gc        /* (gc agFun xs) first groups the list of pairs xs on their */ 
          /* first component, and then applies the aggregation function */ 
          /* agFun to the second component */ 
\end{verbatim}

\subsection{3-ary Operator}

\begin{verbatim}
if        /* if e1 e2 e3 returns e2 if e1 is True and e3 otherwise */
\end{verbatim}

\end{document}
