diff --git a/manual/Makefile b/manual/Makefile new file mode 100644 index 00000000..8023b5c8 --- /dev/null +++ b/manual/Makefile @@ -0,0 +1,18 @@ +.SILENT: + +.PHONY: all clean + +LATEX = pdflatex +PAPER = main_book +OUTPUT = ece252_manual.pdf + +main: + $(LATEX) $(PAPER) + bibtex $(PAPER) + $(LATEX) $(PAPER) + mv $(PAPER).pdf $(OUTPUT) + +clean: + rm -f *~ $(PAPER).dvi $(PAPER).log $(PAPER).pdf $(PAPER).ind $(OUTPUT) + rm -f $(PAPER).bbl $(PAPER).blg $(PAPER).ps $(PAPER).aux + rm -f $(PAPER).out $(PAPER).lot $(PAPER).lof $(PAPER).toc diff --git a/manual/app_group_split_request_form.tex b/manual/app_group_split_request_form.tex new file mode 100644 index 00000000..a9a58032 --- /dev/null +++ b/manual/app_group_split_request_form.tex @@ -0,0 +1,38 @@ +\chapter{Forms} +Lab administration related forms are given in this appendix. +\newpage +\begin{center} +{\bf ECE252 Request to Leave a Project Group Form} +\end{center} + +\begin{table*}[h] + \begin{center} + \begin{tabular}{|p{6cm}|p{6cm}|} + \hline + Name: & \\ \hline + Quest ID: & \\ \hline + Student ID: & \\ \hline + Lab Assignment ID & \\ \hline + Group ID: & \\ \hline + Name of Other Group Members: & \\ \hline + \end{tabular} + \end{center} +\end{table*} + +Provide the reason for leaving the project group here: + +\vspace{9cm} + +\begin{table*} +\begin{center} +\begin{tabular}{p{6cm} p{1cm} p{6cm}} +Signature && Date \\ +\cline{1-1} +\cline{3-3} +\end{tabular} +\end{center} +\end{table*} +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "main_book" +%%% End: diff --git a/manual/bib/bibliography_book.tex b/manual/bib/bibliography_book.tex new file mode 100644 index 00000000..c9cd6b80 --- /dev/null +++ b/manual/bib/bibliography_book.tex @@ -0,0 +1,10 @@ +%========================================================== +% BIBLIOGRAPHY +%========================================================== + +\bibliographystyle{plain} +\bibliography{main} +\renewcommand{\bibname}{References} % for the report or book class +\addcontentsline{toc}{chapter}{\textbf{References}} % for book/report class +%\renewcommand{\refname}{References} % for the article class +%\addcontentsline{toc}{section}{\textbf{References}} % for article class diff --git a/manual/code/linux/HelloWorld/Makefile b/manual/code/linux/HelloWorld/Makefile new file mode 100644 index 00000000..79597f76 --- /dev/null +++ b/manual/code/linux/HelloWorld/Makefile @@ -0,0 +1,5 @@ +helloworld.out: helloworld.o + gcc -o helloworld.out helloworld.o + +helloworld.o: helloworld.c + gcc -c helloworld.c diff --git a/manual/code/linux/HelloWorld/Makefile1 b/manual/code/linux/HelloWorld/Makefile1 new file mode 100644 index 00000000..5e0974d4 --- /dev/null +++ b/manual/code/linux/HelloWorld/Makefile1 @@ -0,0 +1,2 @@ +helloworld.out: helloworld.c + gcc -o helloworld.out helloworld.c diff --git a/manual/code/linux/HelloWorld/Makefile1.tex b/manual/code/linux/HelloWorld/Makefile1.tex new file mode 100644 index 00000000..5e0974d4 --- /dev/null +++ b/manual/code/linux/HelloWorld/Makefile1.tex @@ -0,0 +1,2 @@ +helloworld.out: helloworld.c + gcc -o helloworld.out helloworld.c diff --git a/manual/code/linux/HelloWorld/Makefile2.tex b/manual/code/linux/HelloWorld/Makefile2.tex new file mode 100644 index 00000000..5df6462d --- /dev/null +++ b/manual/code/linux/HelloWorld/Makefile2.tex @@ -0,0 +1,4 @@ +helloworld.out: helloworld.o + gcc -o helloworld.out helloworld.o +helloworld.o: helloworld.c + gcc -c helloworld.c diff --git a/manual/code/linux/HelloWorld/Makefile3.tex b/manual/code/linux/HelloWorld/Makefile3.tex new file mode 100644 index 00000000..01d5329b --- /dev/null +++ b/manual/code/linux/HelloWorld/Makefile3.tex @@ -0,0 +1,9 @@ +all: app.out +app.out: src1.o src2.o + gcc -o app.out src1.o src2.o +src1.o: src1.c + gcc -c src1.c +src2.o: src2.c + gcc -c src2.c +clean: + rm *.o app.out diff --git a/manual/code/linux/HelloWorld/Makefile4.tex b/manual/code/linux/HelloWorld/Makefile4.tex new file mode 100644 index 00000000..85603408 --- /dev/null +++ b/manual/code/linux/HelloWorld/Makefile4.tex @@ -0,0 +1,16 @@ +# Makefile to build app.out +CC=gcc +CFLAGS=-Wall -g +LD=gcc +LDFLAGS=-g + +OBJS=src1.o src2.o + +all: app.out +app.out: $(OBJS) + $(LD) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) +.c.o: + $(CC) $(CFLAGS) -c $< +.PHONY: clean +clean: + rm -f *.o *.out diff --git a/manual/code/linux/HelloWorld/helloworld.c b/manual/code/linux/HelloWorld/helloworld.c new file mode 100644 index 00000000..f47a01a0 --- /dev/null +++ b/manual/code/linux/HelloWorld/helloworld.c @@ -0,0 +1,8 @@ +#include +#include + +int main() +{ + printf("Hello World!\n"); + exit(0); +} diff --git a/manual/code/linux/HelloWorld/main.c b/manual/code/linux/HelloWorld/main.c new file mode 100644 index 00000000..f47a01a0 --- /dev/null +++ b/manual/code/linux/HelloWorld/main.c @@ -0,0 +1,8 @@ +#include +#include + +int main() +{ + printf("Hello World!\n"); + exit(0); +} diff --git a/manual/code/linux/HelloWorld/makefile.tex b/manual/code/linux/HelloWorld/makefile.tex new file mode 100644 index 00000000..79597f76 --- /dev/null +++ b/manual/code/linux/HelloWorld/makefile.tex @@ -0,0 +1,5 @@ +helloworld.out: helloworld.o + gcc -o helloworld.out helloworld.o + +helloworld.o: helloworld.c + gcc -c helloworld.c diff --git a/manual/frontmatter.tex b/manual/frontmatter.tex new file mode 100644 index 00000000..50c17ece --- /dev/null +++ b/manual/frontmatter.tex @@ -0,0 +1,55 @@ +\frontmatter + +% title page, list of tables, list of figures +\input{frontmatter_title} + +%Brief overview of the document, acknowledgement, disclaimer +\chapter{Preface} + +\section*{Who Should Read This Lab Manual?} +This lab manual is written for students who are taking Electrical and Computer Engineering (ECE) Systems Programming and Concurrency course ECE252 in the University of Waterloo. + +\section*{What is in This Lab Manual?} + +The first purpose of this document is to provide the descriptions of each laboratory project. +The second purpose of this document is a quick reference guide of the relevant development tools +for completing laboratory projects. This manual is divided into three parts. + +Part I describes the lab administration policies + +Part II is a set of course laboratory projects as follows. + +\begin{itemize} + \item Lab1: Introduction to systems programming in Linux computing environment + \item Lab2: Multi-threaded concurrency programming with blocking I/O + \item Lab3: Inter-process communication and concurrency control + \item Lab4: Parallel web crawling + \item Lab5: Single-threaded concurrency programming with asynchronous I/O +\end{itemize} + +Part III is a quick reference guide of the Linux software development tools. We will be using Ubuntu 18.04 LTS operating system. Materials in this part needs to be self-studied before lab starts. +The main topics are as follows. + +\begin{itemize} + \item Linux hardware environment + \item Editors + \item Compiler + \item Debugger + \item Utility to automate build + \item Utility for version control +\end{itemize} + + +% A C K N O W L E D G E M E N T S +% ------------------------------- + +\section*{Acknowledgments} + +We are grateful that Professor Patrick Lam shared his ECE459 projects with us. Eric praetzel has provided continuous IT support, which makes the Linux computing environment available to our students. + +We would like to sincerely thank our students who took ECE254 and ECE459 courses in the past few years. They provided constructive feedback every term to make the manual more useful to address problems that students would encounter when working on each lab assignment. + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "main_book" +%%% End: diff --git a/manual/frontmatter_title.tex b/manual/frontmatter_title.tex new file mode 100644 index 00000000..56632bf9 --- /dev/null +++ b/manual/frontmatter_title.tex @@ -0,0 +1,90 @@ +% T I T L E P A G E +% ------------------- +% This file goes along with the master LaTeX file uw-ethesis.tex +% Last updated May 27, 2009 by Stephen Carr, IST Client Services +% The title page is counted as page `i' but we need to suppress the +% page number. We also don't want any headers or footers. +\pagestyle{empty} +\pagenumbering{roman} + +% The contents of the title page are specified in the "titlepage" +% environment. +\begin{titlepage} + \begin{center} + \vspace*{1.0cm} + + \Huge + {\bf Electrical and Computer Engineering (ECE) Systems Programming and Concurrency ECE252 Laboratory Manual} + + \vspace*{1.0cm} + + \normalsize + by \\ + + \vspace*{1.0cm} + + \Large + Yiqing Huang \\ + Jeff Zarnett + + \vspace*{3.0cm} + + \normalsize + Electrical and Computer Engineering Department \\ + University of Waterloo \\ + + \vspace*{2.0cm} +\makeatletter + Waterloo, Ontario, Canada, \@date \\ +\makeatother + + + + \vspace*{1.0cm} + + \copyright\ Y. Huang and J. Zarnett 2019 \\ + \end{center} +\end{titlepage} + +% The rest of the front pages should contain no headers and be numbered using Roman numerals starting with `ii' +\pagestyle{plain} +\setcounter{page}{2} + +\cleardoublepage % Ends the current page and causes all figures and tables that have so far appeared in the input to be printed. +% In a two-sided printing style, it also makes the next page a right-hand (odd-numbered) page, producing a blank page if necessary. +%\newpage + +\cleardoublepage +%\newpage + +% T A B L E O F C O N T E N T S +% --------------------------------- +\tableofcontents +\cleardoublepage +%\newpage + +% L I S T O F T A B L E S +% --------------------------- +\listoftables +\addcontentsline{toc}{chapter}{List of Tables} +\cleardoublepage +%\newpage + +% L I S T O F F I G U R E S +% ----------------------------- +\listoffigures +\addcontentsline{toc}{chapter}{List of Figures} +\cleardoublepage +%\newpage + +% L I S T O F S Y M B O L S +% ----------------------------- +% \renewcommand{\nomname}{Nomenclature} +% \addcontentsline{toc}{chapter}{\textbf{Nomenclature}} +% \printglossary +% \cleardoublepage +% \newpage + +% Change page numbering back to Arabic numerals +\pagenumbering{arabic} + diff --git a/manual/img/img_concatenation.PNG b/manual/img/img_concatenation.PNG new file mode 100644 index 00000000..68ab347a Binary files /dev/null and b/manual/img/img_concatenation.PNG differ diff --git a/manual/img/lab0/MobaXterm/MobaXterm_Login_Password.PNG b/manual/img/lab0/MobaXterm/MobaXterm_Login_Password.PNG new file mode 100644 index 00000000..8b8d46fb Binary files /dev/null and b/manual/img/lab0/MobaXterm/MobaXterm_Login_Password.PNG differ diff --git a/manual/img/lab0/MobaXterm/MobaXterm_Path.PNG b/manual/img/lab0/MobaXterm/MobaXterm_Path.PNG new file mode 100644 index 00000000..e55bd398 Binary files /dev/null and b/manual/img/lab0/MobaXterm/MobaXterm_Path.PNG differ diff --git a/manual/img/lab0/MobaXterm/MobaXterm_Start_Page.PNG b/manual/img/lab0/MobaXterm/MobaXterm_Start_Page.PNG new file mode 100644 index 00000000..5da5bfff Binary files /dev/null and b/manual/img/lab0/MobaXterm/MobaXterm_Start_Page.PNG differ diff --git a/manual/img/lab0/lab0_P_Drive.PNG b/manual/img/lab0/lab0_P_Drive.PNG new file mode 100644 index 00000000..03d1ef10 Binary files /dev/null and b/manual/img/lab0/lab0_P_Drive.PNG differ diff --git a/manual/img/png_chunk_format.PNG b/manual/img/png_chunk_format.PNG new file mode 100644 index 00000000..b77ebec1 Binary files /dev/null and b/manual/img/png_chunk_format.PNG differ diff --git a/manual/img/png_file_format.PNG b/manual/img/png_file_format.PNG new file mode 100644 index 00000000..12d59491 Binary files /dev/null and b/manual/img/png_file_format.PNG differ diff --git a/manual/lab1_intro_linux_sys_prog.tex b/manual/lab1_intro_linux_sys_prog.tex new file mode 100644 index 00000000..f0c1ad27 --- /dev/null +++ b/manual/lab1_intro_linux_sys_prog.tex @@ -0,0 +1,316 @@ +\chapter{Introduction to Systems Programming in Linux Computing Environment} +\section{Introduction} +\subsection{Objectives} +This lab is to introduce system programming in a general Linux Development Environment at ECE Department. After finishing this lab, students will be able to +\begin{itemize} + \item apply basic Linux commands to interact with the Linux system through shell; + \item apply standard Linux C programming tools for system programming and + \item create a program to interact with Linux file systems by applying the relevant system and libray calls. +\end{itemize} + +\subsection{Topics} +Concretely, the lab will cover the following topics: +\begin{itemize} + %\item how to connect to a remote Linux server; + \item Basic Linux commands + \item C programming toolchain including \verb+gcc+, \verb+make+, and \verb+ddd+ + \item Linux manual pages + \item Linux system calls and file I/O library calls to traverse a directory and perform read/write operations on selected files. + \end{itemize} + +\section{Starter Files} + +The starter files are on GitHub at url: \url{http://github.com/yqh/ece252/tree/master/lab1/starter}. +It contains the following sub-directories where we have example code and image files to help you get started: +\begin{itemize} + \item the \href{http://github.com/yqh/ece252/tree/master/lab1/starter/cmd_arg}{cmd\_arg} demonstrates how to capture command line input arguments; + \item the \href{http://github.com/yqh/ece252/tree/master/lab1/starter/images}{images} contains some image files; + \item the \href{http://github.com/yqh/ece252/tree/master/lab1/starter/ls}{ls} demonstrates how to list all files under a directory and obtain file types; + \item the \href{http://github.com/yqh/ece252/tree/master/lab1/starter/png_util}{png\_util} provides a set of utility functions to process a PNG image file; + \item the \href{http://github.com/yqh/ece252/tree/master/lab1/starter/pointer}{pointer} demonstrates how to use pointers to access a C structure; and + \item the \href{http://github.com/yqh/ece252/tree/master/lab1/starter/segfault}{segfault} contains a broken program that has a segmentation fault bug, which you will debug in the last Pre-lab exercise. +\end{itemize} +Using the code in the starter files is permitted and will not be considered as plagiarism. + +\section{Pre-lab Preparation} + +%Read Chapter \ref{ch_linux_env}. +Read the Introduction to ECE Linux Programming Environment supplementary material in Part \ref{part_ref} Chapter \ref{ch_linux_env}. Do the pre-lab exercises in Section \ref{sec_ex1}. Finish the pre-lab programming assignment before your scheduled lab starts (see \ref{sec:lab1_prelab_assignment}). +\subsection{Basic Linux Commands Exercises} +\label{sec_ex1} + +These pre-lab exercises are to practice some basic commands on Linux. +\begin{enumerate} + \item Use the MobaXterm to login onto + \code{eceubuntu.uwaterloo.ca}. You are now inside the Linux shell and in your home directory. The home directory usually has a path name in the format of \verb+/home/username+, where username normally is your UWID. For example, a user with UWID of \verb+jsmith+ has a home directory of \verb+/home/jsmith+. + \item Use the \code{pwd} command to print the full filename of the current working directory. You should see your home directory name printed on the screen. For example: \verb+/home/jsmith+. + \item Use the \code{echo \$HOME} command to print your home directory path name. You will notice that the output matches the \verb+pwd+ output of exercise 2. + \item Use the \code{env} command to list all the environment variables and their values. Note that \verb+HOME+ is one of the many environment variables. + \item One important environment variable is \verb+PATH+. It specifies a set of directories the system searches for executible programs. Use \code{echo \$PATH} to see your \verb+PATH+ environment variable setting. + \item Execute command \code{which ls} to locate the directory the ls command is in. You will notice the directory is listed in \verb+PATH+ environment variable. When you issue a command and get an error message of ``command not found'', it means the command cannot be found after searching all the directories listed in \verb+PATH+ environment variable. A commonly seen error is that a command in your current working directory gives you ``command not found`` error. This is normally due to the fact that the current working directory \verb+.+ or \verb+./+ is not in the PATH. Consequently you need to add the path to the command name for the system to know where the command is. For example \code{./a.out} tells the system to run the command \verb+a.out+ located in the current working directory. + \item Use the \code{ls} command to list all files in your current working + directory. + \item Read the online manual of the \verb+ls+ command by issuing + \code{man ls} command to the shell. + Find out from the manual what options \verb+-l+, \verb+-a+ and \verb+-la+ do. + Execute the \verb+ls+ command with these three options and see the execution results. + \item Create a directory as the work space of labs under your home directory. + Name the newly created directory as \verb+labs+. + Read the man page of the command \verb+mkdir+ to see how to do it. + \item Change directory to the newly create directory of \verb+labs+. + Read the man page of command \verb+cd+ to find out how to change directory. + \item Clone the ece252 lab repository by using the command: \\ + \code{git clone https://github.com/yqh/ece252.git}. + + A new directory named \verb+ece252+ will be created. It has lab manual and starter code of ECE252 labs. + \item Read the man page of the \verb+find+ command by issuing + \code{man find} command to the shell. Read what the \verb+-name+ option does. Use \verb+find+ with the \verb+-name+ option to find all the files with .png file extension in the \verb+$HOME/labs/ece252+ directory. + \item Change directory to where the \verb+WEEF_1.png+ is. Use \code{file WEEF\_1.png} command to obtain the file type and image properties such as dimensions and bit depth. + \item Use \verb+file+ command to obtain the file type information of \verb+Disguise.png+. You should see that this is not an image file though the file exension is .png. Use a text editor to open the file and see the contents. This exercise is to show you that the \verb+file+ command does not obtain the file type information based on the file extension. It looks into the contents of file to extract the file type information + %REV%\footnote{A file has a magic number to indicate its type. The magic number is a sequence of bytes usually appearing near the beginning of the file. The file command checks the magic number. The PNG file's magic number is 89 50 4E 47 OD 0A 1A 0A in hexadecimal format. It is interesting to notice that the second, third and fourth characters are P N and G in ASCII.}. + \footnote{A file has a magic number to indicate its type. The magic number is a sequence of bytes usually appearing near the beginning of the file. The file command checks the magic number. The PNG file's magic number is 89 50 4E 47 in hexadecimal, which is .PNG in ASCII.}. + \item You can use the command \verb+display+ to display an image. For example, to view the \verb+WEEF_1.png+ file, use the command \verb+display WEEF_1.png+. + \item Execute \code{cat red-green-16x16.png} command and you will notice the output are gibberish funny characters. This is because cat displays plain text file in a human readable form, not a binary file. The PNG image file is a binary file. Both vi and emacs have hexadecimal mode which displays the bytes in binary files in a hexadecimal format. They are pretty good for small size binary files. There are also few linux commands that perform hex dump of a file. The \verb+xxd+ is one of them. Try \code{xxd red-green-16x16.png} and see the output. Other similar commands include \verb+od+ and \verb+hexdump+. Refer to the man pages of these commands for detailed usage instructions. + \item The \verb+pngcheck+ command test PNG image files for corruption. Some image viewers are able to display the image even part of the data are corrupted. An image can be opened by an image viewer does not guarantee it is not corrupted. You will notice that the \verb+display+ command will not display a corrupted PNG image file on Linux. But the same corrupted image file most likely can be displayed by Windows Paint program. Execute the following two commands and see what pngcheck output tells you. + \begin{itemize} + \item \code{pngcheck red-green-16x16.png} + \item \code{pngcheck red-green-16x16-corrupted.png} + \end{itemize} + \item Images are binary files. To compare two binary files, we can use the \verb+cmp+ command. Read the man page of the \verb+cmp+ command and especially pay attention to the \verb+-l+ option. Use the \verb+cmp+ command with the \verb+-l+ option to find out which bytes in \verb+red-green-16x16.png+ and \verb+red-green-16x16-corrupted.png+ are different. Another tool is \verb+vbindiff+, which displays and compares hexadecimal file(s). + \item Use gdb or ddd to run the program under \url{http://github.com/yqh/ece252/tree/master/lab1/starter/segfault} directory of the starter code. When the code generates a segmentation fault inside the debugger, run the gdb command \code{where} to see the stack trace and fix the segmentation fault problem of the code. +\end{enumerate} + +\subsection{Pre-lab Assignment} +\label{sec:lab1_prelab_assignment} +We will write some small pieces of code that can be used in the final lab assignment code. We will create a command line program named \verb+pnginfo+ that prints the dimensions of a valid PNG image file and an error message to the standard output if the input file is not a PNG file or is a corrupted PNG file. The command takes one input argument, which is the path name of a file. Both absolute path name and relative path name are accepted. For example, command \code{./pnginfo WEEF\_1.png} will output the following line: +\begin{verbatim} +WEEF_1.png: 450 x 229 +\end{verbatim} +If the input file is not a PNG file. Then output an error message. For example, command \code{./pnginfo Disguise.png} will output the following line: +\begin{verbatim} +Disguise.png: Not a PNG file +\end{verbatim} +If the input file is a PNG file, but certain chunks has CRC error. That is the CRC value in the chunk does not match the CRC value computed by your program, then the program output something similar as what pngcheck does. For example, command \code{./pnginfo red-green-16x16-corrupted.png} will output the following line: +\begin{verbatim} +red-green-16x16-corrupted.png: 16 x 16 +IDAT chunk CRC error: computed 34324f1e, expected dc5f7b84 +\end{verbatim} + +You will find starter files under \href{http://github.com/yqh/ece252/tree/master/lab1/starter/png_util}{png\_util} directory are helpful. To make our pre-lab code reusable in the final lab, we will create two functions. One is \verb+is_png()+ which takes eight bytes and check whether they match the PNG image file signature. Another function is \verb+get_data_IHDR()+ which extracts the image meta information including height and width from a PNG file IHDR chunk. You are free to design the signatures of these two functions so that they will be re-used in your lab1 final solution and future labs 2 and 3 solutions\footnote{lab2 and lab3 are based on the code of lab1}. However in \verb+lab_png.h+ you will find some existing function prototypes that we put there to show one possible function prototype design. Feel free to modify these function prototypes to fit your own design. For computing the CRC of a sequence of bytes, the starter file already provides the crc.c and crc.h and the main.c that demos how to call the crc function to do the computation. +\section{Lab Assignment} +\subsection{Problem statement} + +\begin{wrapfigure}{r}{2.5in} + \includegraphics[width=2in]{img/img_concatenation} + \caption{Image Concatenation Illustration} +\label{fig_img_concatenation} +\end{wrapfigure} + +You are given a directory under which some files are PNG images and some files are not. The directory may contain nested sub-directories\footnote{A nested sub-directory is a sub-directory that may contain many layers of sub-directories.}. All valid PNG images under the given directory are horizontal strips of a bigger whole image. They all have the same width. The height of each image might be different. The PNG images have the naming convention of \verb+*_N.png+, where \verb+N+ is the image strip sequence number and \verb+N=0, 1, 2, ...+. However a file with .png or .PNG extension may not be a real PNG image file. You need to located all the real PNG image files under the given directory first. Then you will concatenate these horizontal strip images sequentially based on the sequence number in the file name to restore the original whole image. The sequence number indicates the order the image should be concatenated from top to bottom. For example, file \verb+img_1.png+ is the first horizontal strip and \verb+img_2.png+ is the second horizontal strip. To concatenate these two strips, the pixel data in \verb+img_1.png+ should be followed immediately by the pixel data in \verb+img_2.png+ file. Figure \ref{fig_img_concatenation} illustrates the concatenation order. + +To solve the problem, first you will create a tool named \verb+findpng+ to search the given directory hierarchy to find all the real PNG files under it. Secondly you will create an image data concatenation tool named \verb+catpng+ to concatenate pixel data of a set of PNG files to form a single PNG image file. The \verb+catpng+ only processes PNG images with the same width in dimension. +\subsection{The findpng command} +The expected behaviour of the \verb+findpng+ is given in the following manual page of the command. +\subsubsection{Man page of findpng} +\subsubsection*{NAME} +\begin{itemize} + \item[]{\bf findpng} - search for PNG files in a directory hierarchy +\end{itemize} +\subsubsection*{SYNOPSIS} +\begin{itemize} + \item[]{\bf findpng} DIRECTORY +\end{itemize} +\subsubsection*{DESCRIPTION} +\begin{itemize} + \item[]Search for PNG files under the directory tree rooted at DIRECTORY and return the search results to the standard output. The command does not follow symbolic links. +\end{itemize} +\subsubsection*{OUTPUT FORMAT} +\begin{itemize} + \item[]The output of search results is a list of PNG file relative path names\footnote{It is relative to the command input directory path name}, one file pathname per line. The order of listing the search results is not specified. If the search result is empty, then output ``findpng: No PNG file found''. +\end{itemize} +\subsubsection*{EXAMPLES} +\begin{itemize} + \item[]{\bf findpng .} + \item[]Find PNG of the current working directory. A non-empty search results might look like the following: + \begin{verbatim} + lab1/sandbox/new_bak.png + lab1/sandbox/t1.png + png_img/rgba_scanline.png + png_img/v1.png + \end{verbatim} + It might also look like the following: + \begin{verbatim} + ./lab1/sandbox/new_bak.png + ./lab1/sandbox/t1.png + ./png_img/rgba_scanline.png + ./png_img/v1.png + \end{verbatim} + An empty search result will look like the following: + \begin{verbatim} + findpng: No PNG file found + \end{verbatim} +\end{itemize} +\subsubsection{Searching PNG files under a given directory} +UNIX file system is organized as a tree. A file has a type. Three file types that this assignment will deal with are regular, directory and symbolic link. A PNG file is a regular file. A directory is a directory file. A link created by \verb+ls -n+ is a symblic link. Read the section 2 of \code{stat} family system calls man page for information about other file types. The \verb+ls/ls_ftype.c+ in the starter code gives a sample program to determine the file type of a given file. Note that the \verb+struct dirent+ returned by the \verb+readdir()+ has a field \verb+d_type+ that also gives the file type information. However it is not supported by all file system types. We will be using eceubuntu machines to test your submission. If you want to use the \verb+d_type+ in your code, make sure you test its behaviour on eceubuntu machines. + +To search all the files under a given directory and its subdirectories, one need to traverse the given directory tree to its leaf nodes. The library call of \code{opendir} returns a directory stream for \code{readdir} to read each entry in a directory. One need to call \code{closedir} to close the directory stream once operations on it is completed. The control flow is to go through each entry in a directory and check the file type. If it is a regular file, then further check whether it is a PNG file by comparing the first 8 bytes with the PNG file header bytes (see Section \ref{subsec_PNG_File_Format}). If it is a directory file, then you need to check files under the sub-directory and repeat what you did in the parent directory. +% The \verb+chdir()+ family system calls change a working directory. +The \verb+ls/ls_fname.c+ in the starter code gives a sample program that lists all file entries of a given directory. +%The \verb+chdir/chdir1.c+ in the starter code gives one example of how to use the \verb+chdir()+ system call. + +Always check the man page of the systems calls and library calls for detailed information. + + +\subsection{The catpng command} + +The expected behaviour of the \verb+catpng+ is given in the following manual page of the command. +\subsubsection{Man page of the catpng} +\subsubsection*{NAME} + \begin{itemize} + \item[]{\bf catpng} - concatenate PNG images vertically to a new PNG named all.png + \end{itemize} +\subsubsection*{SYNOPSIS} + \begin{itemize} + \item[]{\bf catpng} [PNG\_FILE]... + \end{itemize} +\subsubsection*{DESCRIPTION} + \begin{itemize} + \item[]Concatenate PNG\_FILE(s) vertically to all.png, a new PNG file. + \end{itemize} +\subsubsection*{OUTPUT FORMAT} + \begin{itemize} + \item[]The concatenated image is output to a new PNG file with the name of all.png. + \end{itemize} +\subsubsection*{EXAMPLES} +\begin{itemize} +\item[] +\begin{verbatim} +catpng ./img1.png ./png/img2.png +\end{verbatim} +\item[]Concatenate the listed PNG images vertically to all.png. +\end{itemize} + +\subsubsection{File I/O} +There are two sets of functions for file I/O operations under Linux. At system call level, we have the {\em unbufferred I/O} functions: \code{open}, \code{read}, \code{write}, \code{lseek} and \code{close}. At library call level, we have standard I/O functions: \code{fopen}, \code{fread}, \code{fwrite}, \code{fseek} and \code{fclose}. The library is built on top of unbufferred I/O functions. It handles details such as buffer allocation and performing I/O in optimal sized chunks to minimize the number of \code{read} and \code{write} usage, hence is recommended to be used for this lab. + +The \code{fopen} returns a FILE pointer given a file name and the mode. A PNG image file is a binary file, hence when you call \code{fopen}, use mode "\code{rb}" for reading and "\code{wb+}" for reading and writing, where the "b" indicates it is a binary file that we are opening. Read the man page of \code{fopen} for more mode options. + +After the file is opened, use \code{fread} to read the number of bytes from the stream pointed by the FILE pointer returned by \code{fopen}. Each opened file has an internal state of file position indicator. The file position indicator sets to the beginning of the file when it is just opened. The \code{fread} operation will advance the file position indicator by the number of bytes that has been read from the file. The \code{fseek} sets the file position indicator to the user specified location. The \code{fwrite} writes user specified number of bytes to the stream pointed by the FILE pointer. The file position indicator also advances by the number of bytes that has been written. It is important to call \code{fclose} to close the file stream when I/O operations are finished. Failure to do so may result in incomplete files. + +The man pages of the standard I/O library is the main reference for details including function prototypes and how to use them. + +\subsubsection{PNG File Format} +\label{subsec_PNG_File_Format} +In order to finish this assignment, one need to have some understanding of the png file format and how an image is represented in the file. One way to store an image is to use an array of coloured dots referred to as {\em pixels}. A row of pixels within an image is called a {\em scanline}. Pixels are ordered from left-to-right within each scanline. Scanlines appear top-to-bottom in the pixel array. In this assignment, each pixel is represented as four 8-bit +\footnote{Formally, we say the image has a bit depth of 8 bits per sample.} +unsigned integers (ranging from 0 to 255) that specify the red, green, blue and alpha intensity values. This encoding is often referred to as the RGBA encoding. RGB values specify the colour and the alpha value specifies the opacity of the pixel. The size of each pixel is determined by the number of bits per pixel. The dimensions of an image is described in terms of horizontal and vertical pixels. + +% For each PNG pixel, there are five kinds of information namely read, green ,blue, greyscale, and alpha. A {\em channel} is an array of all per-pixel information of one of these five kinds. For example, the red channel is the array of red values of +\begin{figure}[h] +\centering +\subfigure[PNG File Format] { + \includegraphics[width = 3in]{img/png_file_format} + \label{fig_png_file_format} +} +\subfigure[PNG Chunk Format] { + \includegraphics[width = 2.5in]{img/png_chunk_format} + \label{fig_png_chunk_format} +} +\end{figure} + +PNG stands for ``Portable Network Graphics''. It is a computer file format for storing, transmitting and displaying images\cite{Roelofs1999PNG}. A PNG file is a binary file. It starts with an 8-byte header followed by a series of chunks. You will notice the second, third and fourth bytes are the ASCII code of 'P', 'N' and 'G' respectively (see Figure \ref{fig_png_file_format}). + +The first chunk is the IHDR chunk, which contains meta information of the image such as the dimensions of the pixels. The last chunk is always the IEND chunk, which marks the end of the image datastream. In between there is at least one IDAT chunk which contains the compressed filtered pixel array of the image. There are other types chunks that may appear between IHDR chunk and IEND chunk. For all the PNG files we are dealing with in this assignment, we use the format that only has one IHDR chunk, one IDAT chunk and one IEND chunk (see Figure \ref{fig_png_file_format}). + + +Each chunk consists of four parts. A four byte length field, a four byte chunk type code field, the chunk data field whose length is specified in the chunk length field, and a four byte CRC (Cyclic Redundancy Check) field (see Figure \ref{fig_png_chunk_format}). + +The length field stores the length of the data field in bytes. PNG file uses {\em big endian} byte order, which is the network byte order. When we process any PNG data that is more than one byte such as the length field, we need to convert the network byte order to host order before doing arithmetic. The \code{ntohl} and \code{htonl} library calls convert a 32 bit unsigned integer from network order to host order and vice versa respectively. + + + +The chunk type code consists four ASCII character. IHDR, IDAT and IEND are the three chunk type code that this assignment involves with. + +The data field contains the data bytes appropriate to the chunk type. This field can be of zero length. + +The CRC field calculates on the proceeding bytes in the type and data fields of the chunk. Note that the length field is not included in the CRC calculation. The \code{crc} function under the \verb+png_util+ starter code can be used to calculate the CRC value. +\begin{wraptable}{r}{3in} + %\begin{table} + \begin{center} + \begin{tabular}{llc} \toprule + Name & Length & Value \\ \midrule + Width & 4 bytes & N/A\\ \midrule + Height & 4 bytes & N/A\\ \midrule + Bit depth & 1 byte & 8\\ \midrule + Colour type & 1 byte & 6\\ \midrule + Compression method & 1 byte & 0\\ \midrule + Filter method & 1 byte & 0\\ \midrule + Interlace method & 1 byte & 0\\ \bottomrule + \end{tabular} + \caption{IHDR data field and value} + \label{lab1:tb_IHDR_Data} + \end{center} + %\end{table} +\end{wraptable} +The IHDR chunk data field has a fixed length of 13 bytes and they appear in the order as shown in Table \ref{lab1:tb_IHDR_Data}. Width and height are four-byte unsigned integers giving the image dimensions in pixels. You will need these two values to complete this assignment. Bit depth gives the number of bits per sample. In this assignment, all images have a bit depth of 8. Colour type defines the PNG image type. All png images in this assignment have a colour type of 6, which is truecolor with alpha (i.e. RGBA image). The image pixel array data are filtered to prepare for the next step of compression. The Compression method and Filter method bytes encode the methods used. Both only have 0 values defined in the current standard. The Interlace method indicates the transmission order of the image data. 0 (no interlace) and 1 (Adam7 interlace) are the only two defined. In this assignment, all PNG images are non-interlaced. Table \ref{lab1:tb_IHDR_Data} Value column gives the typical IHDR values the PNG images you will be processing. + +The IDAT chunk data field contains compressed filtered pixel data. For each scanline, first an extra byte is added at the very beginning of the pixel array to indicate the filter method used. Filtering is for preparing the next step of compression. For example, if the raw pixel scanline is 4 bytes long, then the scanline after applying filter will be 5 bytes long. This added one byte per scanline will help to achieve better compression result. After all scanlines have be filtered, then the data are compressed according to the compression method encoded in IHDR chunk. The compressed data stream conforms to the zlib 1.0 format. + +The IEND chunk marks the end of the PNG datastream. It has an empty data field. + +\subsubsection{Concatenate the pixel data} +To concatenate two horizontal image strips, the natural way of thinking is to start with the pixel array of each image and then concatenate the two pixel arrays vertically. Then we apply the filter to each scanline. Lastly we compress the filtered pixel array to fill the data field of the new IDAT chunk of the concatenated image. However a simpler way exists. We can start with the filtered pixel data of each image and then concatenate the two chunks of filtered pixel data arrays vertically, then apply the compression method to generate the data field of the new IDAT chunk. + +How do we get filtered pixel data from a PNG IDAT chunk? Recall that the data field in IDAT chunk is compressed data that conforms to zlib format 1.0. We can use zlib functions to uncompress(i.e. inflate) the data. The \code{mem\_inf} in the starter code takes in memory compressed(i.e. deflated) data as input and returns the uncompressed data to a another memory location. For each IDAT chunk you want to concatenate, call this function and stack the returned data in the order you wish and then you have the concatenated filtered pixel array. To create an IDAT chunk, we need to compress the filtered pixel data. The \code{mem\_def} function in the starter code uses the zlib to compress (i.e. deflate) the input in memory data and returns the deflated data. The \verb+png_util+ directory in the starter code demos how to use the aforementioned two functions. + +To create a new PNG for the concatenated images, IHDR chunk also needs to have the new dimension information of the new PNG file. The rest of the fields of IHDR chunk can be kept the same as one of the PNG files to be concatenated. In this assignment, we assume that \verb+catpng+ can only process PNG files whose IHDR chunks only differ in the height field. So the new image will have a different height field, the rest of fields are the same as the input images. + + + +\section{Deliverables} +\subsection{Pre-lab deliverables} +\label{lab1_prelab_deliverable} +Pre-lab is due by the time that your scheduled lab sessions starts. No late submission of pre-lab is accepted. Grace days are not applicable to pre-lab submissions. The following are the steps to create your pre-lab deliverable submission. +\begin{itemize} +\item Create a directory and name it lab1-pre. +\item Put the entire source code with a Makefile under the directory lab1-pre. The Makefile default target includes \verb+pnginfo+. That is command \verb+make+ should generate the aforementioned executable file. We also expect that command \verb+make clean+ will remove the object code and the default target. That is the \verb+.o+ files and the executable file should be removed. +\item Use \verb+zip+ command to zip up the contents of lab1-pre directory and name it lab1-pre.zip. We expect \verb+unzip lab1-pre.zip+ will produce a \verb+lab1-pre+ sub-directory in the current working directory and under the \verb+lab1-pre+ sub-directory is your source code and the Makefile. +\end{itemize} +Submit the \verb+lab1-pre.zip+ file to Lab1 Pre-lab Dropbox in Learn by the time your scheduled lab session starts. The TA on duty will evaluate the pre-lab by letting you demo the submitted program during the scheduled lab session. + +\subsection{Post-lab Deliverables} +Post-lab is due two days after your scheduled lab session at 22:00. Grace days can be used towards late submissions of the post-lab deliverable. The following are the steps to create your post-lab deliverable submission. +\begin{itemize} +\item Create a directory and name it lab1. +\item Put the entire source code with a Makefile under the directory lab1. The Makefile default target includes \verb+catpng+ and \verb+findpng+. That is command \verb+make+ should generate the aforementioned two executable files. We also expect that command \verb+make clean+ will remove the object code and the default target. That is the \verb+.o+ files and the two executable files should be removed. +\item Use \verb+zip+ command to zip up the contents of lab1 directory and name it lab1.zip. We expect \verb+unzip lab1.zip+ will produce a \verb+lab1+ sub-directory in the current working directory and under the \verb+lab1+ sub-directory is your source code and the Makefile. +\end{itemize} +Submit the \verb+lab1.zip+ file to Lab1 Dropbox in Learn. +\section{Marking Rubric} +\begin{table}[ht] +\begin{center} +\begin{tabular}{|p{2cm}|p{2cm}|p{9cm}|} +\hline +Points & Sub-points &Description \\ \hline +15 & & Pre-lab \\ \hline + & 2 & \verb+Makefile+ correctly builds and cleans \\ \hline + & 13& Implementation of \verb+pnginfo+ \\ \hline +% & 3 & \verb+ckpng+ detects a non-png file type \\ \hline +% & 5 & \verb+ckpng+ prints non-corrupted png image dimensions \\ \hline +% & 5 & \verb+ckpng+ detects incorrect CRC of the simple PNG data chunk \\ \hline +85 & & Post-lab \\ \hline + & 10 & Makefile correctly builds and cleans \\ \hline + & 30 & Implementation of \verb+findpng+ \\ \hline + & 45 & Implementation of \verb+catpng+ \\ \hline +\end{tabular} +\caption{Lab1 Marking Rubric} +\label{tb_lab1_rubric} +\end{center} +\end{table} + +Table \ref{tb_lab1_rubric} shows the rubric for marking the lab. Note that if your code generates a segmentation fault, the maximum lab grade you can achieve is 60/100. + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "main_book" +%%% End: diff --git a/manual/lab2_curl_pthreads.tex b/manual/lab2_curl_pthreads.tex new file mode 100644 index 00000000..fab96fe0 --- /dev/null +++ b/manual/lab2_curl_pthreads.tex @@ -0,0 +1,188 @@ +\chapter{Multi-threaded Programming with Blocking I/O} +\label{ch2_pthreads} + +\section{Objectives} +This lab is to learn about and gain practical experience in multi-threaded programming in a general Linux environment. A single-thread implementation using blocking I/O to request a resource across the network is provided. Students are asked to reduce the latency of this operation by sending out multiple requests simultaneously to different machines by using the \verb+pthreads+ library. After this lab, students will have a good understanding of +\begin{itemize} + \item how to design and implement a multi-threaded program by using the \verb+pthreads+ library; and + \item the role mutli-threading plays in reducing the latency of a program. +\end{itemize} + +\section{Starter Files} +The starter files are on GitHub at url: \url{http://github.com/yqh/ece252/tree/master/lab2/starter}. +It contains the following sub-directories where we have example code to help you get started: +\begin{itemize} + \item the \href{http://github.com/yqh/ece252/tree/master/lab2/starter/cURL}{cURL} demonstrates how to use cURL to fetch an image segment from the lab web server; + \item the \href{http://github.com/yqh/ece252/tree/master/lab2/starter/fn_ptr}{fn\_ptr} demonstrates how C function pointers work; + \item the \href{http://github.com/yqh/ece252/tree/master/lab2/starter/getopt}{getopt} demonstrates how to parse command line options; + \item the \href{http://github.com/yqh/ece252/tree/master/lab2/starter/pthreads}{pthreads} demonstrates how to create two threads where each thread takes multiple input parameters and return multiple values; and + \item the \href{http://github.com/yqh/ece252/tree/master/lab2/starter/times}{times} provides helper functions to profile program execution times. +\end{itemize} +Using the code in the starter files is permitted and will not be considered as plagiarism. + +\section{Pre-lab Preparation} +Read the entire lab2 manual to understand what the lab assignment is about. Build the starter code and run the executables. Work through the code and understand what they do and how they work. Create a single-threaded implementation of the \verb+paster+ command (see \ref{sec:man_paster_par}). +Finish the pre-lab deliverable (see Section \ref{sec:lab2-pre-lab-deliverable}). + +%\section{Warm-up Exercises} + +\section{Lab Assignment} +\subsection{Problem Statement} +\label{sec:lab2_problem_statement} +In the previous lab, a set of horizontal strips of a whole PNG image file were stored on a disk and you were asked to restore the whole image from these strips. In this lab, the horizontal image segments are on the web. I have three $400 \times 300$ pictures (whole images) on three web servers. When you ask a web server to send you a picture, the web server crops the picture into fifty +$400 \times 6$ equally sized horizontal strips +\footnote{Each image segment will have a size less than 8KB.}. The web server assigns a sequence number to each strip from top to bottom starting from 0 and increments by 1 +\footnote{The first horizontal strip has a sequence number of 0, the second strip has a sequence number of 1. The sequence number increments by 1 from top to bottom and the last strip has a sequence number of 49.}. Then the web server sleeps for a random time and then sends out a random strip in a simple PNG format that we assumed in lab1. That is the horizontal strip PNG image segment consists one IHDR chunk, one IDAT chunk and one IEND chunk (see Figure \ref{fig_png_file_format}). The PNG segment is an 8-bit RGBA/color image (see Table \ref{lab1:tb_IHDR_Data} for details). The web server uses an HTTP response header that includes the sequence number to tell you which strip it sends to you. The HTTP response header has the format of ``X-Ece252-Fragment: $M$'' where $M \in [0, 49]$. To request a random horizontal strip of picture $N$, where $N\in[1,3]$, use the following URL: +\verb+http://machine:2520/image?img=N+, where + +\texttt{machine} is one of the following: +\begin{itemize} +\item \texttt{ece252-1.uwaterloo.ca}, +\item \texttt{ece252-2.uwaterloo.ca}, and +\item \texttt{ece252-3.uwaterloo.ca}~. +\end{itemize} + +For example, when you request data from the following URL: \\ +\verb+http://ece252-1.uwaterloo.ca:2520/image?img=1+, \\ +you may receive a random horizontal strip of picture 1. Assume this random strip you receive is the third horizontal strip (from top to bottom of the original picture), the received HTTP header will contain ``X-Ece252-Fragment: 2``. The received data will be the image segment in PNG format. +You may use the browser to view a random horizontal strip of the PNG image the server sends. You will notice the same URL displays a different image strip every time you hit enter to refresh the page. Each strip has the same dimensions of $400 \times 6$ pixels and is in PNG format. + +Your objective is to request all horizontal strips of a picture from the server and then concatenate these strips to restore the original picture. Because every time the server sends a random strip, if you use a loop to keep requesting a random strip from a server, you may receive the same strip multiple times before you receive all the fifty distinct strips. Due to the randomness, it will take a variable amount of time to get all the strips you need to restore the original picture. + +\subsection{Requirements} + +Use the \verb+pthreads+ library, design and implement a threaded program to request all image segments from a web server by using blocking I/O and concatenate these segments together to form the whole image. + +The provided starter code \verb+main_write_header_cb.c+ under \verb+cURL+ directory is a single-threaded implementation which uses \verb+libcurl+ blocking I/O function \verb+curl_easy_peform()+to fetch one random horizontal strip of picure $1$ from one of the web servers into memory and then output the received image segment to a PNG file. Your program should repeatedly fetch the image strips until you have them all. Recall because every time you get a random strip, the amount of time to get all the fifty distinct strips of a picture varies. + +A very inefficient approach is to use a single-threaded loop to keep fetching until you get all fifty distinct strips of a picture and paste them together. You will notice the blocking I/O operation is the main cause of the latency. Your program will be blocked while each time waiting for the \verb+curl_easy_perform()+ to finish. One way to reduce the latency of this operation is to send out multiple blocking I/O requests simultaneously (to different machines) by using \verb+pthreads+. You will use this approach to reduce the latency in this lab +\footnote{Asynchronous I/O is another method to reduce the latency and we will explore it in lab5.}. + +Your program should create as many threads as specified by the user command line input, and distribute the work among the three provided servers. Make sure all of your library (standard glibc and libcurl) calls are {\em thread-safe} (for glibc, e.g. \code{man 3 printf} to look at the documentation). Name your executable as \verb+paster+. The behaviour of the command \verb+paster+ is given in the following section. + +The provided three pictures on the server are for you to test your program. Your program should work for all these pictures. You may want to reuse part of your lab1 code to paste the received image segments together. + +\subsection{Man page of paster} +\label{sec:man_paster_par} + +\subsubsection*{NAME} +\begin{itemize} + \item[]{\bf paster} - pasting downloaded png files together + by using multiple threads and blocking I/O through libcurl. +\end{itemize} + +\subsubsection*{SYNOPSIS} + +\begin{itemize} +\item []{\bf paster} [OPTION]... +\end{itemize} + +\subsubsection*{DESCRIPTION} + +\begin{itemize} +\item [] + With no options, the command retrieves all horizontal image segments of picture 1 from \url{http://ece252-1.uwaterloo.ca:2520/image?img=1} and paste all distinct segments received from top to bottom in the order of the image segment sequence number. Output the pasted image to disk and name it all.png. +\item[] {\bf -t}=NUM + \begin{itemize} + \item[] create NUM threads simultaneously requesting random image segments from multiple lab web servers. When this option is not specified, assumes a single-threaded implementation. + \end{itemize} +\item[] {\bf -n}=NUM + \begin{itemize} + \item[] request a random image segment of picture NUM from the web server. Valid values are 1, 2 and 3. Default value is set to 1. + \end{itemize} +\end{itemize} +\subsubsection*{OUTPUT FORMAT} +\begin{itemize} +\item[] The concated image is output to a PNG file with the name of all.png. +\end{itemize} + +\subsubsection*{EXAMPLES} +\begin{itemize} +\item[] +\begin{verbatim} +paster -t 6 -n 2 +\end{verbatim} +Use 6 threads to simultaneously download all image segments of picture 2 from multiple web servers and concatenate these segments to restore picture 2. Output the concatenated picture to disk and name it all.png. +\end{itemize} + +\subsubsection*{EXIT STATUS} +\begin{itemize} +\item[] + {\bf paster} exits with status 0 upon success and nonzero upon error. +\end{itemize} + +\section{Programming Tips} +\label{sec:code-walk-through} + +\subsection{The libcurl API} +Though the image segment download code using \verb+libcurl+ is provided, familiarize yourself with the libcurl API will help you understand the provided code. The libcurl documentation URL is \url{https://curl.haxx.se/libcurl}. The man page of each function in the libcurl API can be found at URL \url{https://curl.haxx.se/libcurl/c/allfuncs.html}. + +Note the provided example \verb+cURL+ code downloads the received image segment to memory and then output the data in memory to a PNG file. The output to a PNG file is just to make it easier for you to view the downloaded image segment to help you understand the example code. However your \verb+paster+ program does not need to output each segment received to a file. An efficient way (i.e. without unnecessary file I/O) is to directly use the received image segment data in memory instead of outputting the data to a file first and then reading the data back from file into memory. + + +\subsubsection{Thread Safety} +\label{sec:libcurl-thread-safety} + +Libcurl is thread safe but there are a few exceptions. The man page of libcurl-thread(3) (see \url{https://curl.haxx.se/libcurl/c/threadsafe.html}) is the ultimate reference. We re-iterate key points from libcurl manual that are relevant to this lab as follows: +\begin{itemize} +\item The same libcurl handle should not be shared in multiple threads. +\item The libcurl is thread safe but does not have internal thread synchronization mechanism. You will need to take care of the thread synchronization. +\end{itemize} + + +\subsection{The pthreads API} +\label{sec:pthreads} +The \verb+pthreads(7)+ man page gives an overview of POSIX threads and should be read. The SEE ALSO section near the bottom of the man page lists functions in the API. The man pages of \verb+pthread_create(3)+, \verb+pthread_join(3)+ and \verb+pthread_exit(3)+ provide detailed information of how to create, join and terminate a thread. + +\subsubsection{The pthread Memory Leak Bug} +\label{sec:pthread-memory-leak} + +There is a known memory leak bug related to \verb+pthread_exit()+. Please refer to \url{https://bugzilla.redhat.com/show_bug.cgi?id=483821} for details. Using \verb+return()+ instead of \verb+pthread_exit()+ will avoid the memory leak bug. + +\section{Deliverables} + +\subsection{Pre-lab deliverables} +\label{sec:lab2-pre-lab-deliverable} +Create a single-threaded implementation of the \verb+paster+ command. Pre-lab is due by the time that your scheduled lab sessions starts. No late submission of pre-lab is accepted. Grace days are not applicable to pre-lab submissions. The following are the steps to create your pre-lab deliverable submission. +\begin{itemize} +\item Create a directory and name it \verb+lab2-pre+. +\item Put the entire source code with a Makefile under the directory \verb+lab2-pre+. The Makefile default target is \verb+paster+. That is command \verb+make+ should generate the \verb+paster+ executable file. We also expect that command \verb+make clean+ will remove the object code and the default target. That is the \verb+.o+ files and the two executable files should be removed. +\item Use \verb+zip+ command to zip up the contents of lab2-pre directory and name it lab2-pre.zip. We expect \verb+unzip lab2-pre.zip+ will produce a \verb+lab2-pre+ sub-directory in the current working directory and under the \verb+lab2-pre+ sub-directory is your source code and the Makefile. +\end{itemize} +Submit the \verb+lab2-pre.zip+ file to Lab2 Pre-lab Dropbox in Learn. +\subsection{Post-lab Deliverables} +\label{sec:lab2-post-lab-deliverable} +Create a multi-threaded implementation of the \verb+paster+ command. +The following are the steps to create your post-lab deliverable submission. +\begin{itemize} +\item Create a directory and name it lab2. +\item Put the entire source code with a Makefile under the directory lab lab2. The Makefile default target is \verb+paster+. That is command \verb+make+ should generate the \verb+paster+ executable file. We also expect that command \verb+make clean+ will remove the object code and the default target. That is the \verb+.o+ files and the executable file should be removed. +\item Use \verb+zip+ command to zip up the contents of lab2 directory and name it lab2.zip. We expect \verb+unzip lab2.zip+ will produce a \verb+lab2+ sub-directory in the current working directory and under the \verb+lab2+ sub-directory is your source code and the Makefile. +\end{itemize} +Submit the \verb+lab2.zip+ file to Lab2 Dropbox in Learn. + + +\section{Marking Rubric} +\begin{table}[ht] +\begin{center} +\begin{tabular}{|p{2cm}|p{2cm}|p{9cm}|} +\hline +Points & Sub-points &Description \\ \hline +30 & & Pre-lab \\ \hline + & 5 & \verb+Makefile+ correctly builds and cleans \\ \hline + & 25 & Implementation of single-threaded \verb+paster+ \\ \hline +70 & & Post-lab \\ \hline + & 5 & Makefile correctly builds and cleans \\ \hline + & 65 & Implementation of multi-threaded \verb+paster+ \\ \hline +\end{tabular} +\caption{Lab2 Marking Rubric} +\label{tb_lab2_rubric} +\end{center} +\end{table} + +Table \ref{tb_lab2_rubric} shows the rubric for marking the lab. + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "main_book" +%%% End: diff --git a/manual/lab3_producer_consumer.tex b/manual/lab3_producer_consumer.tex new file mode 100644 index 00000000..497ab943 --- /dev/null +++ b/manual/lab3_producer_consumer.tex @@ -0,0 +1,286 @@ +\chapter{Interprocess Communication and Concurrency} + +\section{Objectives} + +This lab is to learn about, and gain practical experience in +interprocess communication and concurrency control in a general Linux environment. +%Message passing and shared memory are two methods for different processes to communications . +%For message passing, processes do not share any memory. The operating system message passing facility takes care of shared memory access within its kernel space. +%For shared memory, processes need to take care of the shared memory conflicting operations. +Shared memory allows multiple processes to share a given region of memory. It is the fastest form for different processes to communicate. Processes need to take care of the shared memory conflicting operations. +The operating system provides concurrency control facility such as semaphore API. + +%The choice of implementing the interprocess communication by message passing or shared memory results in different performance. This lab compares the performance of these two different methods by solving a producer consumer problem. + +After this lab, students will be able to +\begin{itemize} +\item design and implement a multi-processes concurrent program by using the producer-consumer pattern; +\item program with + \begin{itemize} + \item the \verb+fork()+ system call to create a new child process; + \item the \verb+wait()+ family system calls to obtain + the status-change information of a child process; + \item the Linux shared memory API to allow processes to communicate; and + \item the Linux semaphore facility to synchronize processes. + %\item the POSIX message queue facility (\verb++) for inter-process communication; + \end{itemize} +\end{itemize} + +\section{Starter Files} +The starter files are on GitHub at url: \url{http://github.com/yqh/ece252/tree/master/lab3/starter}.It contains the following sub-directories where we have example code to help you get started: +\begin{itemize} +\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/fork}{fork} has example code of creating multiple processes and time the total execution time; + it also demonstrate how a zombie process is created when the parent process does not call \code{wait} family calls; +%\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/mqueue}{mqueue} has example code of using POSIX queue API; +\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/sem}{sem} has example code of using POSIX semaphore shared between processes; +\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/shm}{shm} has example code of using System V shared memory; and +\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/cURL_IPC}{cURL\_IPC} has example code of using a shared memory region as a cURL call back function buffer to download one image segment from a lab server by the child process and writing the downloaded image segment to a file by the parent process; and +\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/tools}{tools} has a shell script to compute statistics of timing data and a shell script to clean IPC facilities. +%\item the \href{http://github.com/yqh/ece252/tree/master/lab3/starter/util}{util} has implementation of functions for PNG image processing; +\end{itemize} +The \href{http://github.com/yqh/ece252/tree/master/lab3/starter/lab3_eceubuntu1.csv}{lab3\_eceubunt1.csv} is the template file that you will need for submitting timing results (see Section \ref{sec:lab3:postlab}). +\section{Pre-lab Preparation} +Read the entire lab3 manual to understand what the lab assignment is about. Build and run the starter code to see what they do. You should work through the provided starter code to understand how they work. The following activities will help you to understand the code. + +\begin{enumerate} +\item Execute \code{man fork} to read the man page of \verb+fork(2)+. +%\item Execute \code{man 3 exec} to read the man page of \verb+exec(3)+ family library calls. +\item Execute \code{man 2 wait} to read the man page of \verb+wait(2)+ family system calls. +\item Execute \code{man ps} to read the man page of the \verb+ps+ command. +%\item Execute \code{man mq\_overview} to read the man page of POSIX mqueue API overview. +\item Execute \code{man shm\_overview} to read Linux man page of POSIX shared memory API overview. At the bottom of the man page, it talks about system V shared memory facilities. Read the corresponding man pages of the system V shared memory API. +\item Execute \code{man sem\_overview} to read Linux man page of POSIX semaphore API overview. +\item Execute \code{man ipcs} and \code{man ipcrm} to read the man pages of Linux IPC facility commands. You will find the \code{-s} and the \code{-m} options are helpful in this lab. +\end{enumerate} +Linux man pages are also available on line at \url{https://linux.die.net/}. + +Finish the pre-lab deliverable (see Section \ref{sec:lab3-pre-lab-deliverable}). +The main data structure to represent the fixed size buffer is a queue. A circular queue is one commonly seen implementation of a fixed size buffer if FIFO is required. A stack is another implementation if LIFO is required. You can either create the data structure yourself or use one from an existing library. If you want to explore the C library queue facilities, check out the man pages of insque(3), remque(3) and queue(3). There are example code at the end of the man pages. + +\section{Lab Assignment} +\subsection{The Producer Consumer Problem} +A producer-consumer problem is a classic multi-tasking problem. There are one or more tasks that create data and they are referred to as {\em producers}. There are one or more tasks that use the data and they are referred to as {\em consumers}. We will have a system of $P$ producers and $C$ consumers. Producers and consumers do not necessarily complete their tasks at the same speed. How many producers should be created and how many consumers should be created to achieve maximum latency improvement\footnote{You probably have already noticed in lab2 that once the number of threads reaches a certain number, you reach the maximum performance improvement.}? What if the buffer receiving the produced data has a fixed size? Another problem to think about is that when we fix the number of producers and consumers, how big the bounded buffer size should be? Is it true the bigger the buffer size is, the more latency improvement we will get, or there is a limit beyond which the bigger buffer size will not bring any further latency improvement? We will do some experiments to answer these questions by solving a similar problem that we solved in lab2 with some additional assumptions. + +In lab2 we used multi-threading to download image segments from the web server and then paste all the segments together. This falls into the unbounded buffer producer consumer problem pattern. We can let producers download the image segments (i.e. creating data) and let consumers extract the image pixel data information (i.e. processing data) for future processing. One easy solution to lab 2 (also commonly seen) is to have one thread that does both the producer and the consumer jobs. This implicitly assumes that the number of producers and consumers are equal. But what if data creation and data processing are running at different speeds\footnote{For example, the data processing part could be more involved such as doing some image transformation. It could also be that the network bandwidth is tight or the lab server is slow so that it takes long to download the image segment.}? It may take more time to download data than to process data or vice versa. Then having the same number of producers and consumers are not optimal. In addition, in lab2, we did not restrict the receiving data buffer size. In a real world, resources are limited and the situation that a fixed size of buffer space to receive the incoming data is more realistic. In this lab, we have the additional constraint that the buffer to receive the image segments from the web server has a fixed size. So the problem we are solving is a bounded buffer producer-consumer problem\footnote{Here is another producer consumer problem example: you can think of the producer as a keyboard device driver and the consumer as the application wishing to read keystrokes from the keyboard; in such a scenario the person typing at the keyboard may enter more data than the consuming program wants, or conversely, the consuming program may have to wait for the person to type in characters. This is, however, only one of many cases where producer/consumer scenarios occur, so do not get too tied to this particular usage scenario.}. + + +%In lab2, we pasted the received image segments together to restore the original image. In this lab, we want to transform the image segments received before pasting them together. I have provided a function \code{convert()} in the starter code \verb+util+ sub-directory to do the transformation. Assume you just modify your lab2 code so that after a thread downloading the image segment, it calls the \verb+convert()+ routine and then hand over the transformed image data to a thread that does the pasting. + +% Do we need to grow memory as number of threads grow? Try to find minimum number of threads we need to reach the max speed up. Yes it is random, but statistically there is a limit. Can we improve the efficiency indefinitely? What about the memory footprint? + +\subsection{Problem Statement} +We are still solving an image concatenation problem. The image strips are the same ones that you have seen in lab2. In lab2, the lab web server sleeps a random seconds before it sends a random horizontal strip of an image to the client. In this lab, we have a different server running at port 2530 which sleeps for a fixed time before it sends a specific image strip requested by the client. The deterministic sleep time in the server is to simulate the time to produce the data. The image format sent by the server is still the simple PNG format (see Figure \ref{fig_png_file_format}). The PNG segment is still an 8-bit RGBA/color image (see Table \ref{lab1:tb_IHDR_Data} for details). The web server still uses an HTTP response header that includes the sequence number to tell you which strip it sends to you. The HTTP response header has the format of ``X-Ece252-Fragment: $M$'' where $M \in [0, 49]$. To request a horizontal strip with sequence number $M$ of picture $N$, where $N\in[1, 3]$, use the following URL: +\verb+http://machine:2530/image?img=N&part=M+, where + +\texttt{machine} is one of the following: +\begin{itemize} +\item \texttt{ece252-1.uwaterloo.ca}, +\item \texttt{ece252-2.uwaterloo.ca}, and +\item \texttt{ece252-3.uwaterloo.ca}~. +\end{itemize} + +For example, when you request data from +\url{http://ece252-1.uwaterloo.ca:2530/image?img=1\&part=2}, +you will receive a horizontal image strip with sequence number 2 of picture 1 . The received HTTP header will contain ``X-Ece252-Fragment: 2``. The received data will be the image segment in PNG format. +You may use the browser to view a horizontal strip of the PNG image the server sends. Each strip has the same dimensions of $400 \times 6$ pixels and is in PNG format. + +Your objective is to request all horizontal strips of a picture from the server and then concatenate these strips in the order of the image sequence number from top to bottom to restore the original picture as quickly as possible for a given set of given input arguments specified by the user command. You should name the concatenated image as \verb+all.png+ and output it to the current working directory. + +There are three types of work involved. The first is to download the image segments. The second is to process downloaded image data and copy the processed data to a global data structure for generating the concatenated image. The third is to generate the concatenated all.png file once the global data structure that holds the concatenated image data is filled. + +The producers will make requests to the lab web server and together they will fetch all 50 distinct image segments. Each time an image segment arrives, it gets placed into a fixed-size buffer of size $B$, +% \footnote{If an image segment has already been received before, then the producer should discard it. We do not want to deposit duplicated image segments into the buffer. This implies that you need to establish a mechanism to keep track of which image segment has already been received among producers.} +shared with the consumer tasks. When there are $B$ image segments in the buffer, producers stop producing. When all 50 distinct image segments have been downloaded from the server, all producers will terminate. That is the buffer can take maximum $B$ items, where each item is an image segment. The horizontal image strips sent out by the lab servers are all less than $10,000$ bytes. + +Each consumer reads image segments out of the buffer, one at a time, and then sleeps for $X$ milliseconds specified by the user in the command line\footnote{This is to simulate data processing takes time.}. Then the consumer will process the received data. The main work is to validate the received image segment and then inflate the received IDAT data and copy the inflated data into a proper place inside the memory. +%to change the opacity of the image and then output the image segment to the disk with a naming convention of \verb+part.png+, where \verb+seq+ is the image segment sequence number +%\footnote{For example, image segment with sequence number 2 will generate part2.png file after the convert() routine changes its opacity.}. + +Given that the buffer has a fixed size, $B$, and assuming that $B < 50$, it is possible for the producers to have produced enough image segments that the buffer is filled before any consumer has read any data. If this happens, the producer is blocked, and must wait till there is at least one free spot in the buffer. + +Similarly, it is possible for the consumers to read all of the data from the buffer, and yet more data is expected from the producers. In such a case, the consumer is blocked, and must wait for the producers to deposit one or more additional image segments into the buffer. + +Further, if any given producer or consumer is using the buffer, all other consumers and producers must wait, pending that usage being finished. That is, all access to the buffer represents a critical section, and must be protected as such. + +The program terminates when it finishes outputting the concatenated image segments in the order of the image segment sequence number to a file named all.png. + +Note that there is a subtle but complex issue to solve. Multiple producers are writing to the buffer, thus a mechanism needs to be established to determine whether or not some producer has placed the last image segment into the buffer. Similarly, multiple consumers are reading from the buffer, thus a mechanism needs to be established to determine whether or not some consumer has read out the last image segment from the buffer +\footnote{Due to network transmission has randomness, the order of image segments placed in the buffer may not necessarily be the same order that they have been requested by the producers. The last image segment in the buffer may not necessarily be the image segment with the biggest sequence number. We do not want to request the same image segment twice since this will bring down the performance, so both producers and consumers know the buffer in total will serve 50 image segments.}. + +\subsection*{Requirements} + +Let +$B$ be the buffer size, +$P$ be the number of producers, +$C$ be the number of consumers, +$X$ be the number of milliseconds that a consumer sleeps before it starts to process the image data, and +$N$ be the image number you want to get from the server. +The producer consumer system is called with the execution command syntax of: +\begin{verbatim} + ./paster2

+\end{verbatim} + +The command will execute per the above description and will then print out the time it took to execute. You should measure the time before you create the first process and the time after the last image segment is consumed and the concatenated all.png image is generated. Use the \code{gettimeofday} for time measurement (see starter code under the \verb+fork+ directory) and terminal screen for display. +Thus your last line of output should look like: +\begin{verbatim} + paster2 execution time: