diff -r 7920ca086e5c -r 148155bb9abc documentation/ethercat_doc.tex --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/documentation/ethercat_doc.tex Mon Sep 25 15:55:46 2006 +0000 @@ -0,0 +1,5037 @@ +%------------------------------------------------------------------------------ +% +% IgH EtherCAT Master Documentation +% +% $Id$ +% +%------------------------------------------------------------------------------ + +\documentclass[a4paper,12pt,BCOR6mm,bibtotoc,idxtotoc]{scrbook} + +\usepackage[latin1]{inputenc} +\usepackage[automark,headsepline]{scrpage2} +\usepackage{graphicx} +\usepackage{makeidx} +\usepackage[refpage]{nomencl} +\usepackage{listings} +\usepackage{svn} +\usepackage{textcomp} +\usepackage{url} +\usepackage{hyperref} + +\setlength{\parskip}{0.8ex plus 0.8ex minus 0.5ex} +\setlength{\parindent}{0mm} + +\setcounter{secnumdepth}{\subsubsectionlevel} + +\DeclareFontShape{OT1}{cmtt}{bx}{n} +{ + <5><6><7><8><9><10><10.95><12><14.4><17.28><20.74><24.88>cmttb10 +}{} + +\lstset{basicstyle=\ttfamily\small,numberstyle=\tiny,aboveskip=4mm, + belowskip=2mm,gobble=2,escapechar=`} +\renewcommand\lstlistlistingname{List of Listings} + +\renewcommand\nomname{Glossary} + +\newcommand{\IgH}{\raisebox{-0.7667ex} + {\includegraphics[height=2.2ex]{images/ighsign}}} + +\SVN $Date$ +\SVN $Revision$ + +\makeindex +\makeglossary + +%------------------------------------------------------------------------------ + +\begin{document} + +\pagenumbering{roman} +\pagestyle{empty} + +\begin{titlepage} + \begin{center} + \rule{\textwidth}{1.5mm} + + {\Huge\bf IgH \includegraphics[height=2.4ex]{images/ethercat} + Master 1.1\\[1ex] + Documentation} + + \vspace{1ex} + \rule{\textwidth}{1.5mm} + + \vspace{\fill} + {\Large Florian Pose, \url{fp@igh-essen.com}\\[1ex] + Ingenieurgemeinschaft \IgH} + + \vspace{\fill} + {\Large Essen, \SVNDate\\[1ex] + Revision \SVNRevision} + \end{center} +\end{titlepage} + +%------------------------------------------------------------------------------ + +\tableofcontents +\listoftables +\listoffigures +\lstlistoflistings + +%------------------------------------------------------------------------------ + +\newpage +\pagestyle{scrheadings} + +\section*{Conventions} +\addcontentsline{toc}{section}{Conventions} +\markleft{Conventions} + +The following typographic conventions are used: + +\begin{itemize} +\item \textit{Italic face} is used for newly introduced terms, file + names, parameter names and in-text source code elements. +\item \texttt{Typewriter face} is used for code examples and + command line output. +\item \texttt{\textbf{Bold typewriter face}} is used for user input in + command lines. +\end{itemize} + +Data values and addresses are specified as hexadecimal values with the +prefix 0x. Example: 0x88A4. Unless otherwise noted, address values are +specified as byte addresses. + +Concerning bit operations, the phrase ``setting a bit'', stands for +setting the bit to $1$, ``clearing a bit'' means setting it to $0$, +respectively. + +Function names are always printed with parentheses, but without +parameters. So, if a function \textit{ecrt\_request\_master()} has +empty parentheses, this does not mean, that it has no parameters. + +If shell commands have to be entered, this is marked by a prompt: + +\begin{lstlisting}[gobble=2] + host> +\end{lstlisting} + +Further, if a shell command has to be entered as the superuser, the +prompt ends with a mesh: + +\begin{lstlisting}[gobble=2] + host# +\end{lstlisting} + +%------------------------------------------------------------------------------ + +\chapter{The IgH EtherCAT Master} +\label{chapter:master} +\pagenumbering{arabic} + +This section will first introduce the master's general features and +the concepts used for master development and will then explain the +master's general architecture and offer details of the different +modules. In addition, it will cover state machine definitions, mailbox +protocol implementation and the user space interface. The last section +will deal with timing aspects. + +%------------------------------------------------------------------------------ + +\section{Feature Summary} +\label{sec:summary} +\index{Master!Features} + +The list below gives a short summary of the features of the +implemented EtherCAT master. + +\begin{itemize} +\item The master runs as a kernel module for Linux 2.6. +\item It comes with EtherCAT-capable network driver for RealTek + RTL8139 (and compatible) network interface cards. + \begin{itemize} + \item The Ethernet hardware is operated without interrupts. + \item Drivers for additional Ethernet hardware can easily be + implemented due to a common device interface provided by the + master. + \end{itemize} +\item The master module supports multiple EtherCAT masters on one + machine. +\item The master code supports any Linux realtime extension through + its independent architecture. + \begin{itemize} + \item RTAI\nomenclature{RTAI}{RealTime Application Interface}, + ADEOS\nomenclature{ADEOS}{Adaptive Domain Environment for + Operating Systems}, etc. + \item It runs well even without realtime extensions. + \end{itemize} +\item Common ``realtime interface'' for modules, that want to use + EtherCAT functionality. + \begin{itemize} + \item Synchronous and asynchronous sending and receiving of frames + is supported. + \item Avoidance of unnecessary copy operations for process data. + \end{itemize} +\item \textit{Domains} are introduced, to allow grouping of process + data objects. + \begin{itemize} + \item Handling of multiple domains with different sampling rates. + \item Automatic calculation of process data mapping, FMMU and sync manager + configuration within each domain. + \end{itemize} +\item Communication through serveral finite state machines. + \begin{itemize} + \item Bus monitoring possible during realtime operation. + \item Automatic reconfiguration of slaves on bus power failure + during realtime operation. + \item Controlling of single slaves during realtime operation. + \end{itemize} +\item Master idle mode. + \begin{itemize} + \item Automatic scanning of slaves upon topology changes. + \item Bus visualisation and EoE processing without a realtime module + connected. + \end{itemize} +\item Implementation of the CANopen-over-EtherCAT (CoE) protocol. + \begin{itemize} + \item Configuration of CoE-capable slaves via SDO interface. + \end{itemize} +\item Implementation of the Ethernet-over-EtherCAT (EoE) protocol. + \begin{itemize} + \item Each master creates virtual network devices that are + automatically coupled to EoE-cap\-able slaves found. + \item This implementation natively supports either a switched or a + routed EoE network architecture. + \end{itemize} +\item User space interface via the System Filesystem + (Sysfs)\nomenclature{Sysfs}{System Filesystem}. + \begin{itemize} + \item User space tool for bus visualisation. + \item Slave E$^2$PROM image reading and writing. + \end{itemize} +\item Seamless system integration though LSB\nomenclature{LSB}{Linux + Standard Base} compliance. + \begin{itemize} + \item Master and network device configuration via Sysconfig files. + \item Linux Standard Base compatible init script for master control. + \end{itemize} +\item Virtual read-only network interface for monitoring and debugging + purposes. +\end{itemize} + +%------------------------------------------------------------------------------ + +\section{License} +\label{sec:license} + +The master code is released under the terms and conditions of the GNU +General Public License\index{GPL} \cite{gpl} (version 2). Other +developers, that want to use EtherCAT with Linux systems, are invited +to use the master code or even participate on development. + +%------------------------------------------------------------------------------ + +\section{General Master Architecture} +\label{sec:masterarch} +\index{Master!Architecture} + +The EtherCAT master is integrated into the Linux 2.6 kernel. This was +an early design decision, which has been made for serveral reasons: + +\begin{itemize} +\item Kernel code has significantly better realtime characteristics, + i.~e. less jitter than user space code. It was foreseeable, that a + fieldbus master has a lot of cyclic work to do. Cyclic work is + usually triggered by timer interrupts inside the kernel. The + execution delay of a function that processes timer interrupts is + less, when it resides in kernel space, because there is no need of + time-consuming context switches to a user space process. +\item It was also foreseeable, that the master code has to directly + communicate with the Ethernet hardware. This has to be done in the + kernel anyway (through network device drivers), which is one more + reason for the master code being in kernel space. +\end{itemize} + +A general overview of the master architecture can be seen in +figure~\ref{fig:masterarch}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.9\textwidth]{images/masterarch} + \caption{Master architecture} + \label{fig:masterarch} +\end{figure} + +\paragraph{Master Module} +\index{Master module} + +The EtherCAT master mainly consists of the master module, containing +one or more EtherCAT masters (section~\ref{sec:mastermod}), the +``Device Interface'' (section~\ref{sec:ecdev}) and the ``Realtime +Interface'' (section~\ref{sec:ecrt}). + +\paragraph{Device Modules} +\index{Device modules} + +Furthermore there are EtherCAT-capable network device driver +modules\index{Device modules}, that connect to the EtherCAT master via +the device interface. These modified network drivers can handle both +network devices used for EtherCAT operation and ``normal'' Ethernet +devices. The common case is, that the master module offers a single +EtherCAT master: An EtherCAT-capable network device driver module +connects one network device to this master, that is now able to send +and receive EtherCAT frames, while all other network devices handled +by the network driver get connected to the kernel's network stack as +usual. + +\paragraph{Realtime Modules} + +A ``realtime module''\index{Realtime module} is a kernel module, that +uses the EtherCAT master for cyclic exchange of process data with +EtherCAT slaves. Realtime modules are not part of the EtherCAT master +code\footnote{Although there are serveral examples provided in the + \textit{examples} directory, see chapter~\ref{chapter:usage} for + more information}, so anybody wanting to use the master has to write +one. A realtime module can ``request'' a master through the realtime +interface. If this succeeds, the module has the control over the +master. It can now configure slaves and set up a process data image +(see section~\ref{sec:processdata}) for cyclic exchange. This cyclic +code has to be provided by the realtime module, so it is in hands of +the developer, which mechanism to use for this. Moreover he has to +decide, whether or not using a Linux realtime extension. + +\paragraph{Why ``Realtime'' Module?} + +The name shall not imply, that a linux realtime extension is +mandatory: The master runs well even without realtime extensions, as +section~\ref{sec:mini} shows. However, the code using the master is +time-critical, because process data IO has to be done in cyclic work. +Some EtherCAT slaves support watchdog units, that stop driving the +outputs when process data was not exchanged for some time. So the +names ``realtime interface'' and ``realtime module'' are quite +appropriate. + +%------------------------------------------------------------------------------ + +\subsection{Handling of Process Data} +\label{sec:processdata} + +\paragraph{Process Data Image} +\index{Process data} + +The slaves offer their inputs and outputs by presenting the master +so-called ``Process Data Objects'' (PDOs\index{PDO}). The available +PDOs can be determined by reading out the slave's TXPDO and RXPDO +E$^2$PROM categories. The realtime module can register the PDOs for +data exchange during cyclic operation. The sum of all registered PDOs +defines the ``process data image'', which is exchanged via the +``Logical ReadWrite'' datagrams introduced +in~\cite[section~5.4.2.4]{dlspec}. + +\paragraph{Process Data Domains} +\index{Domain} + +The process data image can be easily managed by creating co-called +``domains'', which group PDOs and allocate the datagrams needed to +exchange them. Domains are mandatory for process data exchange, so +there has to be at least one. They were introduced for the following +reasons: + +\begin{itemize} +\item The maximum size of a ``Logical ReadWrite'' datagram is limited + due to the limited size of an Ethernet frame: The maximum data size + is the Ethernet data field size minus the EtherCAT frame header, + EtherCAT datagram header and EtherCAT datagram footer: $1500 - 2 - + 12 - 2 = 1484$ octets. If the size of the process data image exceeds + this limit, multiple frames have to be sent, and the image has to be + partitioned for the use of multiple datagrams. A domain manages this + automatically. +\item Not every PDO has to be exchanged with the same frequency: The + values of PDOs can vary slowly over time (for example temperature + values), so exchanging them with a high frequency would just waste + bus bandwidth. For this reason, multiple domains can be created, to + group different PDOs and so allow separate exchange. +\end{itemize} + +There is no upper limit for the number of domains, but each domain +occupies one FMMU in each slave involved, so the maximum number of +domains is also limited by the slaves' capabilities. + +\paragraph{FMMU Configuration} +\index{FMMU!Configuration} + +A realtime module can register PDOs for process data exchange. Every +PDO is part of a memory area in the slave's physical memory, that is +protected by a sync manager \cite[section~6.7]{dlspec} for +synchronized access. In order to make a sync manager react on a +datagram accessing its memory, it is necessary to access the last byte +covered by the sync manager. Otherwise the sync manager will not react +on the datagram and no data will be exchanged. That is why the whole +synchronized memory area has to be included into the process data +image: For example, if a certain PDO of a slave is registered for +exchange with a certain domain, one FMMU will be configured to map the +complete sync-manager-protected memory, the PDO resides in. If a +second PDO of the same slave is registered for process data exchange +within the same domain, and this PDO resides in the same +sync-manager-protected memory as the first PDO, the FMMU configuration +is not touched, because the appropriate memory is already part of the +domain's process data image. If the second PDO belongs to another +sync-manager-protected area, this complete area is also included into +the domains process data image. See figure~\ref{fig:fmmus} for an +overview, how FMMU's are configured to map physical memory to logical +process data images. + +\begin{figure}[htbp] + \centering + \includegraphics[width=\textwidth]{images/fmmus} + \caption{FMMU configuration for serveral domains} + \label{fig:fmmus} +\end{figure} + +\paragraph{Process Data Pointers} + +The figure also demonstrates the way, the realtime module can access the +exchanged process data: At PDO registration, the realtime module has +to provide the address of a process data pointer. Upon calculation of +the domain image and allocation of process data memory, this pointer +is redirected to the appropriate location inside the domain's process +data memory and can later be easily dereferenced by the module code. + +%------------------------------------------------------------------------------ + +\subsection{Operation Modes} +\index{Master modes} + +The EtherCAT master has serveral modes of operation: + +\begin{description} +\item[Orphaned Mode] This mode takes effect, when the master has no + EtherCAT-capable network device connected. No bus communication is + possible, so this mode is not of further interest. +\item[Idle Mode]\index{Idle mode} takes effect when the master is + unused (i.~e. there is no realtime module, that reserved the + master). In this case, the master has the ability to scan the bus by + itsself and generously allow bus access from user space. This mode + is meant for maintenance and visualisation. +\item[Operation Mode]\index{Operation mode} The master is reserved for + exclusive access by a realtime module. In this mode, the master is + adjusted for availability and monitoring. Access from user space is + very restrictive and mostly limited to reading direction. + +\end{description} + +Figure~\ref{fig:modes} shows the three modes and the possible mode +transitions. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.9\textwidth]{images/modes} + \caption{Master modes and transitions} + \label{fig:modes} +\end{figure} + +\subsubsection{Idle Mode} +\index{Idle mode} + +The master enters idle mode upon connection of a device module (see +section~\ref{sec:device}) or releasing by a realtime module. The +master owns a kernel workqueue and a suitable work structure, which is +used to cyclically process the ``Idle state machine'' (see +section~\ref{sec:fsm-idle}). This state machine automatically scans +the bus for slaves (and re-scans upon topology changes), configures +slaves for idle operation and executes pending operations from the +user space interface (for example E$^2$PROM writing). On device +disconnection or realtime request, the idle mode is stopped by +cancelling the work and flushing the workqueue. + +\subsubsection{Operation Mode} +\index{Operation mode} + +Operation mode is entered when a realtime module requests the master. +The idle mode is stopped and the bus is scanned by getting the number +of slaves and executing the ``Slave scan state machine'' (see +section~\ref{sec:fsm-scan}) for each slave. The master is now ready to +create domains and accept PDO registrations and slave configurations. +After that, cyclic communication can be done by the realtime module. + +\paragraph{Master Phases} + +Every realtime module should use the master in three phases: + +\begin{enumerate} +\item \textit{Startup} - The master is requested and the bus is + validated. Domains are created and PDOs are registered. Slave + configurations are applied. +\item \textit{Operation} - Cyclic code is run, process data is + exchanged and the master state machine is executed. +\item \textit{Shutdown} - Cyclic code is stopped and the master + is released. +\end{enumerate} + +%------------------------------------------------------------------------------ + +\section{Device Modules} +\label{sec:device} +\index{Device modules} + +Device modules are network device driver modules that handle Ethernet +devices, which the master can use to connect to an EtherCAT bus. + +Section~\ref{sec:networkdrivers} offers an overview of general Linux +network driver modules, while section~\ref{sec:requirements} will show +the requirements to an EtherCAT-enabled network driver. Finally, +sections~\ref{sec:seldev} to~\ref{sec:patching} show how to fulfill +these requirements and implement such a driver module. + +%------------------------------------------------------------------------------ + +\subsection{Network Driver Basics} +\label{sec:networkdrivers} +\index{Network drivers} + +EtherCAT relies on Ethernet hardware and the master needs a physical +Ethernet device to communicate with the bus. Therefore it is necessary +to understand how Linux handles network devices and their drivers, +respectively. + +\paragraph{Tasks of a Network Driver} + +Network device drivers handle the lower two layers of the OSI model, +that is the physical layer and the data-link layer. A network device +itself natively handles the physical layer issues: It represents the +hardware to connect to the medium and to send and receive data in the +way, the physical layer protocol describes. The network device driver +is responsible for getting data from the kernel's networking stack and +forwarding it to the hardware, that does the physical transmission. +If data is received by the hardware respectively, the driver is +notified (usually by means of an interrupt) and has to read the data +from the hardware memory and forward it to the network stack. There +are a few more tasks, a network device driver has to handle, including +queue control, statistics and device dependent features. + +\paragraph{Driver Startup} + +Usually, a driver searches for compatible devices on module loading. +For PCI drivers, this is done by scanning the PCI bus and checking for +known device IDs. If a device is found, data structures are allocated +and the device is taken into operation. + +\paragraph{Interrupt Operation} +\index{Interrupt} + +A network device usually provides a hardware interrupt that is used to +notify the driver of received frames and success of transmittion, or +errors, respectively. The driver has to register an interrupt service +routine (ISR\index{ISR}\nomenclature{ISR}{Interrupt Service Routine}), +that is executed each time, the hardware signals such an event. If the +interrupt was thrown by the own device (multiple devices can share one +hardware interrupt), the reason for the interrupt has to be determined +by reading the device's interrupt register. For example, if the flag +for received frames is set, frame data has to be copied from hardware +to kernel memory and passed to the network stack. + +\paragraph{The net\_device structure} +\index{net\_device} + +The driver registers a \textit{net\_device} structure for each device +to communicate with the network stack and to create a ``network +interface''. In case of an Ethernet driver, this interface appears as +\textit{ethX}, where X is a number assigned by the kernel on +registration. The \textit{net\_device} structure receives events +(either from user space or from the network stack) via serveral +callbacks, which have to be set before registration. Not every +callback is mandatory, but for reasonable operation the ones below are +needed in any case: + +\begin{description} +\item[int (*open)(struct net\_device *)] This function is called when + network communication has to be started, for example after a command + \textit{ifconfig ethX up} from user space. Frame reception has to be + enabled by the driver. +\item[int (*stop)(struct net\_device *)] The purpose of this callback + is to ``close'' the device, i.~e. make the hardware stop receiving + frames. +\item[int (*hard\_start\_xmit)(struct sk\_buff *, struct net\_device + *)] This function is cal\-led for each frame that has to be + transmitted. The network stack passes the frame as a pointer to an + \textit{sk\_buff} structure (``socket buffer''\index{Socket buffer}, + see below), which has to be freed after sending. +\item[struct net\_device\_stats *(*get\_stats)(struct net\_device *)] + This call has to return a pointer to the device's + \textit{net\_device\_stats} structure, which permanently has to be + filled with frame statistics. This means, that everytime a frame is + received, sent, or an error happened, the appropriate counter in + this structure has to be increased. +\end{description} + +The actual registration is done with the \textit{register\_netdev()} +call, unregistering is done with \textit{unregister\_netdev()}. + +\paragraph{The netif Interface} +\index{netif} + +All other communication in the direction interface $\to$ network stack +is done via the \textit{netif\_*} calls. For example, on successful +device opening, the network stack has to be notified, that it can now +pass frames to the interface. This is done by calling +\textit{netif\_start\_queue()}. After this call, the +\textit{hard\_start\_xmit()} callback can be called by the network +stack. Furthermore a network driver usually manages a frame +transmission queue. If this gets filled up, the network stack has to +be told to stop passing further frames for a while. This happens with +a call to \textit{netif\_stop\_queue()}. If some frames have been +sent, and there is enough space again to queue new frames, this can be +notified with \textit{netif\_wake\_queue()}. Another important call is +\textit{netif\_receive\_skb()}\footnote{This function is part of the + NAPI (``New API''), that replaces the ``old'' kernel 2.4 technique + for interfacing to the network stack (with \textit{netif\_rx()}). + NAPI is a technique to improve network performance on Linux. Read + more in + http://www.cyberus.ca/\textasciitilde{}hadi/usenix-paper.tgz}: It +passes a frame to the network stack, that was just received by the +device. Frame data has to be packed into a so-called ``socket +buffer'' for that (see below). + +\paragraph{Socket Buffers} +\index{Socket buffer} + +Socket buffers are the basic data type for the whole network stack. +They serve as containers for network data and are able to quickly add +data headers and footers, or strip them off again. Therefore a socket +buffer consists of an allocated buffer and serveral pointers that mark +beginning of the buffer (\textit{head}), beginning of data +(\textit{data}), end of data (\textit{tail}) and end of buffer +(\textit{end}). In addition, a socket buffer holds network header +information and (in case of received data) a pointer to the +\textit{net\_device}, it was received on. There exist functions that +create a socket buffer (\textit{dev\_alloc\_skb()}), add data either +from front (\textit{skb\_push()}) or back (\textit{skb\_put()}), +remove data from front (\textit{skb\_pull()}) or back +(\textit{skb\_trim()}), or delete the buffer (\textit{kfree\_skb()}). +A socket buffer is passed from layer to layer, and is freed by the +layer that uses it the last time. In case of sending, freeing has to +be done by the network driver. + +%------------------------------------------------------------------------------ + +\subsection{EtherCAT Network Drivers} +\label{sec:requirements} + +There are a few requirements for Ethernet network devices to function +as EtherCAT devices, when connected to an EtherCAT bus. + +\paragraph{Dedicated Interfaces} + +For performance and realtime purposes, the EtherCAT master needs +direct and exclusive access to the Ethernet hardware. This implies +that the network device must not be connected to the kernel's network +stack as usual, because the kernel would try to use it as an ordinary +Ethernet device. + +\paragraph{Interrupt-less Operation} +\index{Interrupt} + +EtherCAT frames travel through the logical EtherCAT ring and are then +sent back to the master. Communication is highly deterministic: A +frame is sent and will be received again after a constant time. +Therefore, there is no need to notify the driver about frame +reception: The master can instead query the hardware for received +frames. + +Figure~\ref{fig:interrupt} shows two workflows for cyclic frame +transmittion and reception with and without interrupts. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.8\textwidth]{images/interrupt} + \caption{Interrupt Operation versus Interrupt-less Operation} + \label{fig:interrupt} +\end{figure} + +In the left workflow ``Interrupt Operation'', the data from the last +cycle is first processed and a new frame is assembled with new +datagrams, which is then sent. The cyclic work is done for now. +Later, when the frame is received again by the hardware, an interrupt +is triggered and the ISR is executed. The ISR will fetch the frame +data from the hardware and initiate the frame dissection: The +datagrams will be processed, so that the data is ready for processing +in the next cycle. + +In the right workflow ``Interrupt-less Operation'', there is no +hardware interrupt enabled. Instead, the hardware will be polled by +the master by executing the ISR. If the frame has been received in the +meantime, it will be dissected. The situation is now the same as at +the beginning of the left workflow: The received data is processed and +a new frame is assembled and sent. There is nothing to do for the rest +of the cycle. + +The interrupt-less operation is desirable, because there is simply no +need for an interrupt. Moreover hardware interrupts are not conducive +in improving the driver's realtime behaviour: Their undeterministic +incidences contribute to increasing the jitter. Besides, if a realtime +extension (like RTAI) is used, some additional effort would have to be +made to priorize interrupts. + +\paragraph{Ethernet and EtherCAT Devices} + +Another issue lies in the way Linux handles devices of the same type. +For example, a PCI\nomenclature{PCI}{Peripheral Component + Interconnect, Computer Bus} driver scans the PCI bus for devices it +can handle. Then it registers itself as the responsible driver for all +of the devices found. The problem is, that an unmodified driver can +not be told to ignore a device because it will be used for EtherCAT +later. There must be a way to handle multiple devices of the same +type, where one is reserved for EtherCAT, while the other is treated +as an ordinary Ethernet device. + +For all this reasons, the author has decided that the only acceptable +solution is to modify standard Ethernet drivers in a way that they +keep their normal functionality, but gain the ability to treat one or +more of the devices as EtherCAT-capable. + +Below are the advantages of this solution: + +\begin{itemize} +\item No need to tell the standard drivers to ignore certain devices. +\item One networking driver for EtherCAT and non-EtherCAT devices. +\item No need to implement a network driver from scratch and running + into issues, the former developers already solved. +\end{itemize} + +The chosen approach has the following disadvantages: + +\begin{itemize} +\item The modified driver gets more complicated, as it must handle + EtherCAT and non-EtherCAT devices. +\item Many additional case differentiations in the driver code. +\item Changes and bugfixes on the standard drivers have to be ported + to the Ether\-CAT-capable versions from time to time. +\end{itemize} + +%------------------------------------------------------------------------------ + +\subsection{Device Selection} +\label{sec:seldev} + +After loading the master module, at least one EtherCAT-capable network +driver module has to be loaded, that connects one of its devices to +the master. To specify an EtherCAT device and the master to connect +to, all EtherCAT-capable network driver modules should provide two +module parameters: + +\begin{description} +\item[ec\_device\_index] PCI device index of the device that is + connected to the EtherCAT bus. If this parameter is left away, all + devices found are treated as ordinary Ethernet devices. Default: + $-1$ +\item[ec\_master\_index] Index of the master to connect to. Default: + $0$ +\end{description} + +The following command loads the EtherCAT-capable RTL8139 device +driver, telling it to handle the second device as an EtherCAT device +and connecting it to the first master: + +\begin{lstlisting} + host# `\textbf{modprobe ec\_8139too ec\_device\_index=1}` +\end{lstlisting} + +Usually, this command does not have to be entered manually, but is +called by the EtherCAT init script. See section~\ref{sec:init} for +more information. + +%------------------------------------------------------------------------------ + +\subsection{The Device Interface} +\label{sec:ecdev} +\index{Device interface} + +An anticipation to the section about the master module +(section~\ref{sec:mastermod}) has to be made in order to understand +the way, a network device driver module can connect a device to a +specific EtherCAT master. + +The master module provides a ``device interface'' for network device +drivers. To use this interface, a network device driver module must +include the header +\textit{devices/ecdev.h}\nomenclature{ecdev}{EtherCAT Device}, coming +with the EtherCAT master code. This header offers a function interface +for EtherCAT devices which is explained below. All functions of the +device interface are named with the prefix \textit{ecdev}. + +\paragraph{Device Registration} + +A network device driver can connect a physical device to an EtherCAT +master with the \textit{ecdev\_register()} function. + +\begin{lstlisting}[language=C] + ec_device_t *ecdev_register(unsigned int master_index, + struct net_device *net_dev, + ec_isr_t isr, + struct module *module); +\end{lstlisting} + +The first parameter \textit{master\_index} must be the index of the +EtherCAT master to connect to (see section~\ref{sec:mastermod}), +followed by \textit{net\_dev}, the pointer to the corresponding +net\_device structure, which represents the network device to connect. +The third parameter \textit{isr} must be a pointer to the interrupt +service routine (ISR\index{ISR}) handling the device. The master will +later execute the ISR in order to receive frames and to update the +device status. The last parameter \textit{module} must be the pointer +to the device driver module, which is usually accessible via the macro +\textit{THIS\_MODULE} (see next paragraph). On success, the function +returns a pointer to an \textit{ec\_device\_t} object, which has to be +specified when calling further functions of the device interface. +Therefore the device module has to store this pointer for future use. +In error case, the \textit{ecdev\_register()} returns \textit{NULL}, +which means that the device could not be registered. The reason for +this is printed to \textit{syslog}\index{syslog}. In this case, the +device module is supposed to abort the module initialisation and let +the \textit{insmod} command fail. + +\paragraph{Implicit Dependencies} + +The reason for the module pointer has to be specified at device +registration is a non-trivial one: The master has to know about the +module, because there will be an implicit dependency between the +device module and a later connected realtime module: When a realtime +module connects to the master, the use count of the master module will +be increased, so that the master module can not be unloaded for the +time of the connection. This is reasonable, and so automatically done +by the kernel. The kernel knows about this dependency, because the +realtime module uses kernel symbols provided by the master module. +Moreover it is mandatory, that the device module can be unloaded +neither, because it is implicitely used by the realtime module, too. +Unloading it would lead to a fatal situation, because the master would +have no device to send and receive frames for the realtime module. +This dependency can not be detected automatically, because the +realtime module does not use any symbols of the device module. +Therefore the master explicitly increments the use counter of the +connected device module upon connection of a realtime module and +decrements it, if the realtime module disconnects. In this manner, it +is impossible to unload a device module while the master is in use. +This is done with the kernel function pair \textit{try\_module\_get()} +\index{try\_module\_get@\textit{try\_module\_get()}} and +\textit{module\_put()} \index{module\_put@\textit{module\_put()}}. The +first one increases the use count of a module and only fails, if the +module is currenly being unloaded. The last one decreases the use +count again and never fails. Both functions take a pointer to the +module as their argument, which the device module therefore has to +specify upon device registration. + +\paragraph{Device Unregistering} + +The unregistration of a device is usually done in the device module's +cleanup function, by calling the \textit{ecdev\_unregister()} function +and specifying the master index and a pointer to the device object +again. + +\begin{lstlisting}[language=C] + void ecdev_unregister(unsigned int master_index, + ec_device_t *device); +\end{lstlisting} + +This function can fail too (if the master index is invalid, or the +given device was not registered), but due to the fact, that this +failure can not be dealt with appropriately, because the device module +is unloading anyway, the failure code would not be of any interest. So +the function has a void return value. + +\paragraph{Starting the Master} + +When a device has been initialized completely and is ready to send and +receive frames, the master has to be notified about this by calling +the \textit{ecdev\_start()} function. + +\begin{lstlisting}[language=C] + int ecdev_start(unsigned int master_index); +\end{lstlisting} + +The master will then enter ``Idle Mode'' and start scanning the bus +(and possibly handling EoE slaves). Moreover it will make the bus +accessible via Sysfs interface and react to user interactions. The +function takes one parameter \textit{master\_index}, which has to be +the same as at the call to \textit{ecdev\_register()}. The return +value will be non-zero if the starting process failed. In this case +the device module is supposed to abort the init sequence and make the +init function return an error code. + +\paragraph{Stopping the Master} + +Before a device can be unregistered, the master has to be stopped by +calling the \textit{ecdev\_stop()} function. It will stop processing +messages of EoE slaves and leave ``Idle Mode''. The only parameter is +\textit{master\_index}. This function can not fail. + +\begin{lstlisting}[language=C] + void ecdev_stop(unsigned int master_index); +\end{lstlisting} + +A subsequent call to \textit{ecdev\_unregister()} will now unregister +the device savely. + +\paragraph{Receiving Frames} + +The interrupt service routine handling device events usually has a +section where new frames are fetched from the hardware and forwarded +to the kernel network stack via \textit{netif\_receive\_skb()}. For an +EtherCAT-capable device, this has to be replaced by calling the +\textit{ecdev\_receive()} function to forward the received data to the +connected EtherCAT master instead. + +\begin{lstlisting}[language=C] + void ecdev_receive(ec_device_t *device, + const void *data, + size_t size); +\end{lstlisting} + +This function takes 3 arguments, a pointer to the device object +(\textit{device}), a pointer to the received data, and the size of the +received data. The data range has to include the Ethernet headers +starting with the destination address and reach up to the last octet +of EtherCAT data, excluding the FCS. Most network devices handle the +FCS in hardware, so it is not seen by the driver code and therefore +doesn't have to be cut off manually. + +\paragraph{Handling the Link Status} + +Information about the link status (i.~e. if there is a carrier signal +detected on the physical port) is also important to the master. This +information is usually gathered by the ISR and should be forwarded to +the master by calling the \textit{ecdev\_link\_state()} function. The +master then can react on this and warn the realtime module of a lost +link. + +\begin{lstlisting}[language=C] + void ecdev_link_state(ec_device_t *device, + uint8_t new_state); +\end{lstlisting} + +The parameter \textit{device} has to be a pointer to the device object +returned by \textit{ecdev\_\-register()}. With the second parameter +\textit{new\_state}, the new link state is passed: 1, if the link went +up, and 0, if it went down. + +%------------------------------------------------------------------------------ + +\subsection{Patching Network Drivers} +\label{sec:patching} +\index{Network drivers} + +This section will demonstrate, how to make a standard Ethernet driver +EtherCAT-capable. The below code examples are taken out of the +modified RealTek RTL8139 driver coming with the EtherCAT master +(\textit{devices/8139too.c}). The driver was originally developed by +Donald Becker, and is currently maintained by Jeff Garzik. + +Unfortunately, there is no standard procedure to enable an Ethernet +driver for use with the EtherCAT master, but there are a few common +techniques, that are described in this section. + +\begin{enumerate} +\item A first simple rule is, that \textit{netif\_*()}-calls must be + strictly avoided for all EtherCAT devices. As mentioned before, + EtherCAT devices have no connection to the network stack, and + therefore must not call its interface functions. +\item Another important thing is, that EtherCAT devices should be + operated without interrupts. So any calls of registering interrupt + handlers and enabling interrupts at hardware level must be avoided, + too. +\item The master does not use a new socket buffer for each send + operation: Instead there is a fix one allocated on master + initialization. This socket buffer is filled with an EtherCAT frame + with every send operation and passed to the + \textit{hard\_start\_xmit()} callback. For that it is necessary, + that the socket buffer is not be freed by the network driver as + usual. +\end{enumerate} + +As mentioned before, the driver will handle both EtherCAT and ordinary +Ethernet devices. This implies, that for each device-dependent +operation, it has to be checked if an EtherCAT device is involved, or +just an Ethernet device. For means of simplicity, this example driver +will only handle one EtherCAT device. This makes the case +differentiations easier. + +\paragraph{Global Variables} + +First of all, there have to be additional global variables declared, +as shown in the listing: + +\begin{lstlisting}[language=C,numbers=left] + static int ec_device_index = -1; + static int ec_device_master_index = 0; + static ec_device_t *rtl_ec_dev; + struct net_device *rtl_ec_net_dev = NULL; +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1} -- \textcircled{\tiny 2}] To + comply to the requirements for parameters of EtherCAT device modules + described in section~\ref{sec:seldev}, there have to be additional + parameter variables: \textit{ec\_\-device\_\-index} holds the index + of the EtherCAT device and defaults to $-1$ (no EtherCAT device), + while \textit{ec\_device\_master\_index} stores index of the master, + the single device will be connected to. Default: $0$ +\item[\normalfont\textcircled{\tiny 3}] \textit{rtl\_ec\_dev} will be + the pointer to the later registered RealTek EtherCAT device, which + can be used as a parameter for device methods. +\item[\normalfont\textcircled{\tiny 4}] \textit{rtl\_ec\_net\_dev} is + a pointer to the \textit{net\_device} structure of the dedicated + device and is set while scanning the PCI bus and finding the device + with the specified index. This is done inside the + \textit{pci\_module\_init()} function executed as the first thing on + module loading. +\end{description} + +\paragraph{Module Initialization} + +Below is the (shortened) coding of the device driver's module init +function: + +\begin{lstlisting}[language=C,numbers=left] + static int __init rtl8139_init_module(void) + { + if (pci_module_init(&rtl8139_pci_driver) < 0) { + printk(KERN_ERR "Failed to init PCI mod.\n"); + goto out_return; + } + + if (rtl_ec_net_dev) { + printk(KERN_INFO "Registering" + " EtherCAT device...\n"); + if (!(rtl_ec_dev = + ecdev_register(ec_device_master_index, + rtl_ec_net_dev, + rtl8139_interrupt, + THIS_MODULE))) { + printk(KERN_ERR "Failed to reg." + " EtherCAT device!\n"); + goto out_unreg_pci; + } + + printk(KERN_INFO "Starting EtherCAT" + " device...\n"); + if (ecdev_start(ec_device_master_index)) { + printk(KERN_ERR "Failed to start" + " EtherCAT device!\n"); + goto out_unreg_ec; + } + } else { + printk(KERN_WARNING "No EtherCAT device" + " registered!\n"); + } + + return 0; + + out_unreg_ec: + ecdev_unregister(ec_device_master_index, rtl_ec_dev); + out_unreg_pci: + pci_unregister_driver(&rtl8139_pci_driver); + out_return: + return -1; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] This call initializes all + RTL8139-compatible devices found on the pci bus. If a device with + index \textit{ec\_device\_index} is found, a pointer to its + \textit{net\_device} structure is stored in + \textit{rtl\_ec\_net\_dev} for later use (see next listings). +\item[\normalfont\textcircled{\tiny 8}] If the specified device was + found, \textit{rtl\_ec\_net\_dev} is non-zero. +\item[\normalfont\textcircled{\tiny 11}] The device is connected to + the specified master with a call to \textit{ecdev\_register()}. If + this fails, module loading is aborted. +\item[\normalfont\textcircled{\tiny 23}] The device registration was + successful and the master is started. This can fail, which aborts + module loading. +\item[\normalfont\textcircled{\tiny 29}] If no EtherCAT device was + found, a warning is output. +\end{description} + +\paragraph{Device Searching} + +During the PCI initialization phase, a variable \textit{board\_idx} is +increased for each RTL8139-compatible device found. The code below is +executed for each device: + +\begin{lstlisting}[language=C,numbers=left] + if (board_idx == ec_device_index) { + rtl_ec_net_dev = dev; + strcpy(dev->name, "ec0"); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1}] The device with the specified + index will be the EtherCAT device. +\end{description} + +\paragraph{Avoiding Device Registration} + +Later in the PCI initialization phase, the net\_devices get +registered. This has to be avoided for EtherCAT devices and so this is +a typical example for an EtherCAT case differentiation: + +\begin{lstlisting}[language=C,numbers=left] + if (dev != rtl_ec_net_dev) { + i = register_netdev(dev); + if (i) goto err_out; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1}] If the current net\_device is + not the EtherCAT device, it is registered at the network stack. +\end{description} + +\paragraph{Avoiding Interrupt Registration} + +In the next two listings, there is an interrupt requested and the +device's interrupts are enabled. This also has to be encapsulated by +if-clauses, because interrupt operation is not wanted for EtherCAT +devices. + +\begin{lstlisting}[language=C,numbers=left] + if (dev != rtl_ec_net_dev) { + retval = request_irq(dev->irq, rtl8139_interrupt, + SA_SHIRQ, dev->name, dev); + if (retval) return retval; + } +\end{lstlisting} + +\begin{lstlisting}[language=C,numbers=left] + if (dev != rtl_ec_net_dev) { + /* Enable all known interrupts by setting + the interrupt mask. */ + RTL_W16(IntrMask, rtl8139_intr_mask); + } +\end{lstlisting} + +\paragraph{Frame Sending} + +The listing below shows an exerpt of the function representing the +\textit{hard\_start\_xmit()} callback of the net\_device. + +\begin{lstlisting}[language=C,numbers=left] + /* Note: the chip doesn't have auto-pad! */ + if (likely(len < TX_BUF_SIZE)) { + if (len < ETH_ZLEN) + memset(tp->tx_buf[entry], 0, ETH_ZLEN); + skb_copy_and_csum_dev(skb, tp->tx_buf[entry]); + if (dev != rtl_ec_net_dev) { + dev_kfree_skb(skb); + } + } else { + if (dev != rtl_ec_net_dev) { + dev_kfree_skb(skb); + } + tp->stats.tx_dropped++; + return 0; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 6} + \textcircled{\tiny 10}] The + master uses a fixed socket buffer for transmission, which is reused + and may not be freed. +\end{description} + +\paragraph{Frame Receiving} + +During ordinary frame reception, a socket buffer is created and filled +with the received data. This is not necessary for an EtherCAT device: + +\begin{lstlisting}[language=C,numbers=left] + if (dev != rtl_ec_net_dev) { + /* Malloc up new buffer, compatible with net-2e. */ + /* Omit the four octet CRC from the length. */ + + skb = dev_alloc_skb (pkt_size + 2); + if (likely(skb)) { + skb->dev = dev; + skb_reserve(skb, 2); /* 16 byte align + the IP fields. */ + eth_copy_and_sum(skb, &rx_ring[ring_off + 4], + pkt_size, 0); + skb_put(skb, pkt_size); + skb->protocol = eth_type_trans(skb, dev); + + dev->last_rx = jiffies; + tp->stats.rx_bytes += pkt_size; + tp->stats.rx_packets++; + + netif_receive_skb (skb); + } else { + if (net_ratelimit()) + printk(KERN_WARNING + "%s: Memory squeeze, dropping" + " packet.\n", dev->name); + tp->stats.rx_dropped++; + } + } else { + ecdev_receive(rtl_ec_dev, + &rx_ring[ring_offset + 4], pkt_size); + dev->last_rx = jiffies; + tp->stats.rx_bytes += pkt_size; + tp->stats.rx_packets++; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 28}] If the device is an EtherCAT + device, no socket buffer is allocated. Instead a pointer to the data + (which is still in the device's receive ring) is passed to the + EtherCAT master. Unnecessary copy operations are avoided. +\item[\normalfont\textcircled{\tiny 30} -- \textcircled{\tiny 32}] The + device's statistics are updated as usual. +\end{description} + +\paragraph{Link State} + +The link state (i.~e. if there is a carrier signal detected on the +receive port) is determined during execution of the ISR. The listing +below shows the different processing for Ethernet and EtherCAT +devices: + +\begin{lstlisting}[language=C,numbers=left] + if (dev != rtl_ec_net_dev) { + if (tp->phys[0] >= 0) { + mii_check_media(&tp->mii, netif_msg_link(tp), + init_media); + } + } else { + void __iomem *ioaddr = tp->mmio_addr; + uint16_t link = RTL_R16(BasicModeStatus) + & BMSR_LSTATUS; + ecdev_link_state(rtl_ec_dev, link ? 1 : 0); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] The ``media check'' is done + via the media independent interface (MII\nomenclature{MII}{Media + Independent Interface}), a standard interface for Fast Ethernet + devices. +\item[\normalfont\textcircled{\tiny 7} -- \textcircled{\tiny 10}] For + EtherCAT devices, the link state is fetched manually from the + appropriate device register, and passed to the EtherCAT master by + calling \textit{ecdev\_\-link\_\-state()}. +\end{description} + +\paragraph{Module Cleanup} + +Below is the module's cleanup function: + +\begin{lstlisting}[language=C,numbers=left] + static void __exit rtl8139_cleanup_module (void) + { + printk(KERN_INFO "Cleaning up RTL8139-EtherCAT" + " module...\n"); + + if (rtl_ec_net_dev) { + printk(KERN_INFO "Stopping device...\n"); + ecdev_stop(ec_device_master_index); + printk(KERN_INFO "Unregistering device...\n"); + ecdev_unregister(ec_device_master_index, + rtl_ec_dev); + rtl_ec_dev = NULL; + } + + pci_unregister_driver(&rtl8139_pci_driver); + + printk(KERN_INFO "RTL8139-EtherCAT module" + " cleaned up.\n"); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 6}] Stopping and unregistration is + only done, if a device was registered before. +\item[\normalfont\textcircled{\tiny 8}] The master is first stopped, + so it does not access the device any more. +\item[\normalfont\textcircled{\tiny 10}] After this, the device is + unregistered. The master is now ``ophaned''. +\end{description} + +%------------------------------------------------------------------------------ + +\section{The Master Module} +\label{sec:mastermod} +\index{Master module} + +The EtherCAT master is designed to run as a kernel module. Moreover +the master kernel module \textit{ec\_master} can handle multiple +masters at the same time: The number of masters has to be passed to +the module with the parameter \textit{ec\_master\_count}, that +defaults to $1$. A certain master can later be addressed by its index. +For example, if the master module has been loaded with the command + +\begin{lstlisting} + host# `\textbf{modprobe ec\_master ec\_master\_count=2}` +\end{lstlisting} + +the two masters can be addressed by their indices 0 and 1 respectively +(see figure~\ref{fig:masters}). This master index mandatory for +certain functions of the master interfaces. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.5\textwidth]{images/masters} + \caption{Multiple masters in one module} + \label{fig:masters} +\end{figure} + +\paragraph{Master Log Messages} + +The master module gives information about it's state and events via +the syslog interface. The module loading command above should result +in the following syslog messages: + +\begin{lstlisting} + EtherCAT: Master driver, 1.1 (stable) - rev. 513, + compiled by fp at Aug 09 2006 09:43:50 + EtherCAT: Initializing 2 EtherCAT master(s)... + EtherCAT: Initializing master 0. + EtherCAT: Initializing master 1. + EtherCAT: Master driver initialized. +\end{lstlisting} + +The master provides information about it's version number, subversion +revision number and compile information, like the date of compilation +and the user, who compiled. All messages are prefixed either with +\texttt{EtherCAT:}, \texttt{EtherCAT WARNING:} or \texttt{EtherCAT + ERROR:}, which makes searching the logs easier. + +%------------------------------------------------------------------------------ + +\subsection{Class Reference} +\label{sec:classes} + +This section is not intended to be a complete reference of master +classes and functions\footnote{The comprehensive master reference can + be obtained at http://etherlab.org/download/download-en.html}, but +will give a general survey of the master's classes, and how they +interact. + +Figure~\ref{fig:uml-all} shows an UML class diagram of the master +classes. + +\begin{figure}[htbp] + \centering + \includegraphics[width=\textwidth]{images/uml-all} + \caption{UML class diagram with associations} + \label{fig:uml-all} +\end{figure} + +The following subsections introduce serveral classes with their +attributes and methods. + +%------------------------------------------------------------------------------ + +\subsubsection{The Master Class} +\label{sec:class-master} +\index{Master!Class} + +Figure~\ref{fig:uml-master} shows an UML class diagram of the master +class. There is a short explanation of the attributes and methods +below. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.8\textwidth]{images/uml-master} + \caption{Master UML class diagram} + \label{fig:uml-master} +\end{figure} + +\paragraph{Master Attributes} + +\begin{description} +\item[list] is a listhead structure that is needed to manage the list + of masters in the master module (see section~\ref{sec:mastermod}). +\item[reserved] is a flag, that marks the master as reserved for a + realtime module, so that a call to \textit{ecrt\_request\_master()} + fails, if another module is already using the master. +\item[index] contains the number of the master. The first master will + get index 0, the second index 1, and so on. +\item[kobj] In order to make the master object available via Sysfs + (see section~\ref{sec:sysfs}), this structure is needed inside the + master object (see section~\ref{sec:sysfs}). +\item[slaves] is the list of slaves. It consists of objects of the + \textit{ec\_slave\_t} class (see section~\ref{sec:class-slave}). +\item[slave\_count] is the number of slaves in the list. +\item[device] points to the network device that is used by the master + to send and receive frames (see section~\ref{sec:class-device}). It + is \textit{NULL}, if no device is connected. +\item[datagram\_queue] is a list of datagrams (see + section~\ref{sec:class-datagram}) that have to be sent by the + master, or have already been sent and wait to be received again. + Upon reception or error, the datagrams are dequeued. +\item[datagram\_index] contains the index value for the next datagram. + The master stores this incrementing index into every datagram, to + make it easier to assign a received datagram to the one sent before. +\item[domains] contains the list of domains created by the realtime + module (section~\ref{sec:class-domain}). +\item[debug\_level] controls, how much debugging output is printed by + the master: 0 means no debugging output, 1 means to output certain + executing marks and actions, and 2 means to output frame contents in + addition. This value can be changed at runtime via the Sysfs + interface (see section~\ref{sec:sysfs}). +\item[stats] is a statistics object that contains certain counters + (like the number of missed frames). These statistics are output on + demand, but at most once a second. +\item[workqueue] is the kernel workqueue used for idle mode. +\item[idle\_work] is the work object, that is queued. +\item[fsm] The attribute \textit{fsm} represents the master's finite + state machine, that does all the slave processing. See + sections~\ref{sec:class-fsm} and~\ref{sec:fsm} for further details. +\item[mode] contains the current master mode, if it is orphaned, idle, + or in operation mode. +\item[eoe\_timer] is the kernel timer used for EoE\index{EoE} + processing. +\item[eoe\_running] marks the state of EoE processing. +\item[eoe\_handlers] is the list of EoE handlers (see + section~\ref{sec:class-eoe}). +\item[internal\_lock] is a spinlock used in idle mode, that controls + the concurrency of the idle and EoE processes. +\item[request\_cb] The ``request lock'' callback function, the master + has to provide for foreign instances, which want to access the + master (see section~\ref{sec:concurr}). +\item[release\_cb] The callback function that will release the master + lock. +\item[cb\_data] This value will be passed as an argument to each + callback. +\item[eeprom\_write\_enable] flag can be written via Sysfs to enable + the general writing of E$^2$PROM contents. +\end{description} + +\paragraph{Public Master Methods} + +\begin{description} +\item[ec\_master\_init()] is the master's constructor. It initializes + all attributes, creates the workqueue, creates EoE handlers and the + state machine object, and adds the kernel object to the Sysfs + hierarchy. +\item[ec\_master\_clear()] is the destructor and undoes all these + actions. +\item[ec\_master\_reset()] clears the master, but initializes it + again. This is needed, when a realtime module disconnects: Slaves + and other attributes are cleared and are later rebuilt by the idle + process. +\item[ec\_master\_idle\_start/stop()] These methods enable or disable + the idle process. +\item[ec\_master\_eoe\_start/stop()] These methods do the same for the + EoE timer. +\item[ec\_master\_receive\_datagrams()] This method is called by the + device, which uses it to pass received frames to the master. The + frame is dissected and the contained datagrams are assigned to the + datagram objects in the datagram queue, which are dequeued on + reception or error. +\item[ec\_master\_queue\_datagram()] This method is used to queue a + new datagram for sending and receiving. +\item[ec\_master\_output\_stats()] This method is cyclically called to + output a summary of the \textit{stats} attribute at most once a + second. +\item[ec\_master\_clear\_slaves()] clears the list of slaves. This is + needed on connection/disconnection of a realtime module or at a + topology change in idle mode, when all slaves objects are rebuilt. +\end{description} + +\paragraph{Private Master Methods} + +A few of a master's methods are private, meaning, that they can only +be called from other master methods: + +\begin{description} +\item[ec\_master\_send\_datagrams()] searches the datagram queue for + unsent datagrams, allocates frames to send them, does the actual + sending and marks the datagrams as sent. +\item[ec\_master\_idle\_run()] is the work function for the idle mode. + It executes the idle state machine, described in + section~\ref{sec:fsm-idle}. +\item[ec\_master\_eoe\_run()] is called by the EoE timer and is + responsible for communicating with EoE-capable slaves. See + section~\ref{sec:eoeimp} for more information. +\end{description} + +\paragraph{Master Methods (Realtime Interface)} + +The master methods belonging to the Eth\-er\-CAT realtime +interface\index{ecrt@\textit{ecrt}}\nomenclature{ecrt}{EtherCAT + Realtime Interface} begin with the prefix \textit{ecrt} instead of +\textit{ec}. The functions of the realtime interface are explained in +section~\ref{sec:ecrt-master}. + +%------------------------------------------------------------------------------ + +\subsubsection{The Slave Class} +\label{sec:class-slave} +\index{Slave!Class} + +Figure~\ref{fig:uml-slave} shows an UML class diagram of the slave +class. There is a short explanation of the attributes and methods +below. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.8\textwidth]{images/uml-slave} + \caption{Slave UML class diagram} + \label{fig:uml-slave} +\end{figure} + +\paragraph{Slave Attributes} + +\begin{description} +\item[list] The master holds a slave list, therefore the slave class + must contain this structure used as an anchor for the linked + list. +\item[kobj] This pointer serves as base object for the slave's Sysfs + representation. +\item[master] is the pointer to the master owning this slave object. +\item[ring\_position] is the logical position in the logical ring + topology. +\item[station\_address] is the configured station address. This is + always the ring position~+~$1$). +\item[coupler\_index] is the index of the last bus coupler. +\item[coupler\_subindex] is the slave's position, counted from the + last bus coupler. See section~\ref{sec:addr} for more information. +\item[base\_*] These attributes contain base information about the + slave, that are read from the ``DL Information'' attribute. +\item[dl\_*] These fields store information of the ``DL Status'' + attribute, for example states of the the communication ports. +\item[sii\_*] These attributes contain values from the ``Slave + Information Interface'' \cite[section~6.4]{dlspec}, mostly identity + and mailbox information, but also the list of sync manager + configurations and PDOs. +\item[registered] This flag is set, if one or more PDOs of the slave + have been registered for process data exchange. Otherwise a warning + is output, because the slave is unused. +\item[fmmus] Is an array of FMMU configurations, that have to be + applied to the slave. +\item[fmmu\_count] contains number of FMMUs used. +\item[eeprom\_*] These fields contain E$^2$PROM contents and the + extracted category information \cite[section~5.4]{alspec}. +\item[new\_eeprom\_data] If this pointer is not \textit{NULL}, it + points to new E$^2$PROM contents, that have to be written to the + slave. +\item[new\_eeprom\_size] This field represents the size of the new + E$^2$PROM data. +\item[requested\_state] is the requested slave state. +\item[current\_state] is the current slave state. +\item[error\_flag] is used by the operation and idle state machines + to indicate, that a state transisition has failed and should not be + tried again until an external event happens. +\item[online] This flag contains the online state of the slave (i.~e. + if it currently responds to the masters commands). Changes of the + online state are always reported. +\item[varsize\_fields] is only suitable for slaves that provide PDOs + of variable size (like slaves that manage a sub-fieldbus) and + contains information about what size this fields actually should + have. +\end{description} + +\paragraph{Public Slave Methods} + +\begin{description} +\item[ec\_slave\_init()] The slave's constructor. +\item[ec\_slave\_clear()] The slave's destructor. +\item[ec\_prepare\_fmmu()] prepares an FMMU configuration. The FMMU is + configured for a certain sync manager and domain. +\item[ec\_fetch\_*()] Serveral methods to extract information of the + E$^2$PROM category contents. +\item[ec\_slave\_locate\_string()] extracts a string out of a STRING + category and allocates string memory. +\item[ec\_slave\_calc\_sync\_size()] calculates the size of sync + manager contents, because they can be variable due to variable-sized + PDOs. +\item[ec\_slave\_info()] This method prints all slave information into + a buffer for Sysfs reading. +\item[ec\_slave\_mbox\_*()] These functions prepare datagrams for + mailbox communication, or process mailbox responses, respectively. +\end{description} + +\paragraph{Private Slave Methods} + +\begin{description} +\item[ec\_slave\_write\_eeprom()] This function accepts E$^2$PROM data + from user space, does a quick validation of the contents and + schedules them for writing through the idle state machine. +\end{description} + +\paragraph{Slave Methods (Realtime Interface)} + +\begin{description} +\item[ecrt\_slave\_conf\_sdo*()] These methods accept SDO + configurations, that are applied on slave activation (i.~e. + everytime the slave is configured). The methods differ only in the + data size of the SDO (8, 16 or 32 bit). +\item[ecrt\_slave\_pdo\_size()] This method specifies the size of a + variable-sized PDO. +\end{description} + +%------------------------------------------------------------------------------ + +\subsubsection{The Device Class} +\label{sec:class-device} +\index{Device!Class} + +The device class is responsible for communicating with the connected +EtherCAT-enabled network driver. Figure~\ref{fig:uml-device} shows its +UML class diagram. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.3\textwidth]{images/uml-device} + \caption{Device UML class diagram} + \label{fig:uml-device} +\end{figure} + +\paragraph{Device Attributes} + +\begin{description} +\item[master] A pointer to the master, which owns the device object. +\item[dev] This is the pointer to the \textit{net\_device} structure + of the connected network device driver. +\item[open] This flag stores, if the network device is ``opened'' and + ready for transmitting and receiving frames. +\item[tx\_skb] The transmittion socket buffer. Instead of allocating a + new socket buffer for each frame, the same socket buffer is recycled + and used for every frame. +\item[isr] The pointer to the network device's interrupt service + routine. \textit{ec\_isr\_t} is a type definition in the device + interface, which looks like below: + \begin{lstlisting}[gobble=4,language=C] + typedef irqreturn_t (*ec_isr_t)(int, void *, + struct pt_regs *); + \end{lstlisting} +\item[module] A pointer to the network driver module, to increase and + decrease the use counter (see paragraph ``Implicit Dependencies'' in + section~\ref{sec:ecdev}). +\item[link\_state] The current link state. It can be 0 ``down'' or 1 + ``up''. +\item[dbg] Every device objects contains a debugging interface (see + sectios~\ref{sec:class-debug} and~\ref{sec:debug}). +\end{description} + +\paragraph{Public Device Methods} + +\begin{description} +\item[ec\_device\_init()] The device constructor. +\item[ec\_device\_clear()] The device destructor. +\item[ec\_device\_open()] ``Opens'' the device for transmittion and + reception of frames. This is equivalent to the \textit{ifconfig up} + command for ordinary Ethernet devices. +\item[ec\_device\_close()] Stops frame transmittion and reception. + This is equivalent to the \textit{ifconfig down} command for + ordinary Ethernet devices. +\item[ec\_device\_call\_isr()] Calls the interrupt service routine of + the device. +\item[ec\_device\_tx\_data()] Returns a pointer into the memory of the + transmittion socket buffer \textit{tx\_skb}. This is used by the + master to assemble a new EtherCAT frame. +\item[ec\_device\_send()] Sends an assembled frame by passing it to + the device's \textit{hard\_\-start\_\-xmit()} callback. +\end{description} + +\paragraph{Device Methods (Device Interface)} + +The device methods belonging to the device interface are explained in +section~\ref{sec:ecdev}. + +%------------------------------------------------------------------------------ + +\subsubsection{The Datagram Class} +\label{sec:class-datagram} +\index{Datagram!Class} + +So send and receive a datagram, an object of the +\textit{ec\_datagram\_t} class is needed. It can be initialized with a +datagram type \cite[section~5.4]{dlspec} and length (optionally filled +with data) and appended to the master's datagram queue. +Figure~\ref{fig:uml-datagram} shows its UML class diagram. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.3\textwidth]{images/uml-datagram} + \caption{Datagram UML class diagram} + \label{fig:uml-datagram} +\end{figure} + +\paragraph{Datagram Attributes} + +\begin{description} +\item[list] This attribute is needed to make a list of datagrams, as + used in the domain class (see section~\ref{sec:class-domain}). +\item[queue] This attribute is the anchor to the master's datagram + queue, which is implemented as a linked list. +\item[type] The datagram type. \textit{ec\_\-datagram\_\-type\_\-t} is + an enumeration, which can have the values + \textit{EC\_\-DATAGRAM\_\-APRD}, \textit{EC\_\-DATAGRAM\_\-APWR}, + \textit{EC\_\-DATAGRAM\_\-NPRD}, \textit{EC\_\-DATAGRAM\_\-NPWR}, + \textit{EC\_\-DATAGRAM\_\-BRD}, \textit{EC\_\-DATAGRAM\_\-BWR} or + \textit{EC\_\-DATAGRAM\_\-LRW}. +\item[address] The slave address. For all addressing schemes take 4 + bytes, \textit{ec\_address\_t} is a union type: + \begin{lstlisting}[gobble=4,language=C] + typedef union { + struct { + uint16_t slave; /**< configured or + autoincrement + address */ + uint16_t mem; /**< physical memory + address */ + } physical; /**< physical address */ + uint32_t logical; /**< logical address */ + } ec_address_t; + \end{lstlisting} +\item[data] The actual data of the datagram. These are either filled + in before sending (at writing access) or are inserted by the + adressed slave(s) (at reading access). In any case, the data memory + must be dynamically allocated. Besides, this can be done before + cyclic processing with the \textit{ec\_datagram\_prealloc()} method + (see below). +\item[mem\_size] The size of the allocated memory, \textit{data} + points to. +\item[data\_size] The size of the actual data in the \textit{data} + memory. +\item[index] The sequential EtherCAT datagram index. This value is set + by the master on sending, to easier assign a received datagram to a + queued datagram object. +\item[working\_counter] The working counter of the datagram. This is + set to zero on sending and filled with the real value of the working + counter on datagram reception. +\item[state] The state of the datagram. + \textit{ec\_\-datagram\_\-state\_\-t} is an enumeration and can be + \textit{EC\_\-DATA\-GRAM\_\-INIT}, + \textit{EC\_\-DATA\-GRAM\_\-QUEU\-ED}, + \textit{EC\_\-DATA\-GR\-AM\_\-SEN\-T}, + \textit{EC\_\-DATA\-GRAM\_\-REC\-EIVED}, + \textit{EC\_\-DATA\-GRAM\_\-TIMED\_\-OUT} or + \textit{EC\_\-DA\-TA\-GRAM\_\-ERR\-OR}. +\item[t\_sent] This attribute is set to the timestamp, when the + datagram was sent, to later detect a timeout. +\end{description} + +\paragraph{Public Datagram Methods} + +\begin{description} +\item[ec\_datagram\_init()] The datagram's constructor. +\item[ec\_datagram\_clear()] The datagram's destructor. +\item[ec\_datagram\_prealloc()] Allocates memory for the datagram + data. This is especially needed, if the datagram structure will + later be used in a context, where no dynamic memory allocation is + allowed. +\item[ec\_datagram\_nprd()] Initializes a ``Node-Addressed Physical + Read'' datagram \cite[section~5.4.1.2]{dlspec}. +\item[ec\_datagram\_npwr()] Initializes a ``Node-Addressed Physical + Write'' datagram \cite[section~5.4.2.2]{dlspec}. +\item[ec\_datagram\_aprd()] Initializes a ``Auto-Increment Physical + Read'' datagram \cite[section~5.4.1.1]{dlspec}. +\item[ec\_datagram\_apwr()] Initializes a ``Auto-Increment Physical + Write'' datagram \cite[section~5.4.2.1]{dlspec}. +\item[ec\_datagram\_brd()] Initializes a ``Broadcast Read'' datagram + \cite[section~5.4.1.3]{dlspec}. +\item[ec\_datagram\_bwr()] Initializes a ``Broadcast Write'' datagram + \cite[section~5.4.2.3]{dlspec}. +\item[ec\_datagram\_lrw()] Initializes a ``Logical ReadWrite'' + datagram \cite[section~5.4.3.4]{dlspec}. +\end{description} + +%------------------------------------------------------------------------------ + +\subsubsection{The Domain Class} +\label{sec:class-domain} +\index{Domain!Class} + +The domain class encapsules PDO registration and management of the +process data image and its exchange. The UML class diagram can be seen +in figure~\ref{fig:uml-domain}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.4\textwidth]{images/uml-domain} + \caption{Domain UML class diagram} + \label{fig:uml-domain} +\end{figure} + +\paragraph{Domain Attributes} + +\begin{description} +\item[kobj] This \textit{kobject} structure is needed for the Sysfs + representation of the domain. +\item[list] The master manages a list of domains, so this list anchor + is needed. +\item[index] The domain's index. The first domain will get index 0, + the second index 1, and so on. +\item[master] A pointer to the master owning the domain. +\item[data\_size] The size of the domain's process data image. +\item[datagram] A linked list with the datagram objects, the domain + needs for process data exchange (see + section~\ref{sec:class-datagram}). +\item[base\_address] This attribute stores the logical offset, to + which the domain's process data are mapped. +\item[response\_count] The sum of the datagrams' working counters at + the last process data exchange. Changes are always reported. +\item[data\_regs] The (linked) list of PDO registrations. The realtime + module requests the exchange of certain PDOs and supplies the + address of process data pointers, that will later point to the + respective locations in the process data image. These ``data + registrations'' are saved in the \textit{data\_regs} list. +\item[working\_counter\_changes] This field stores the number of + working counter changes since the last notification. This helps to + reduce syslog output in case of frequent changes. +\item[t\_last] The timestamp of the last working counter change + notification. +\end{description} + +\paragraph{Public Domain Methods} + +\begin{description} +\item[ec\_domain\_init()] The domain's constructor. +\item[ec\_domain\_clear()] The domain's destructor. +\item[ec\_domain\_alloc()] Allocates the process data image and the + respective datagrams based on the process data registrations. +\item[ec\_domain\_queue()] Queues the domain's datagrams for exchange + via the master. +\end{description} + +\paragraph{Private Domain Methods} + +\begin{description} +\item[ec\_domain\_reg\_pdo\_entry()] This method is used to do a PDO + registration. It finds the appropriate sync manager covering the PDO + data, calculates its offset in the sync-manager-protected memory and + prepares the FMMU configurations for the related slave. Then the PDO + registration is appended to the list. +\item[ec\_domain\_clear\_data\_regs()] Clearing all process data + registrations is needed in serveral places and therefore has been + sourced out to an own method. +\item[ec\_domain\_add\_datagram()] This methods allocates a datagram + and appends it to the list. This is done during domain allocation. +\end{description} + +\paragraph{Domain Methods (Realtime Interface)} + +The domain methods belonging to the realtime interface are introduced +in section~\ref{sec:ecrt-domain}. + +%------------------------------------------------------------------------------ + +\subsubsection{The Finite State Machine Class} +\label{sec:class-fsm} +\index{FSM!Class} + +This class encapsules all state machines, except the EoE state +machine. Its UML class diagram can be seen in +figure~\ref{fig:uml-fsm}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.9\textwidth]{images/uml-fsm} + \caption{Finite State Machine UML class diagram} + \label{fig:uml-fsm} +\end{figure} + +\paragraph{FSM Attributes} + +\begin{description} +\item[master] A pointer to the master owning the FSM object. +\item[slave] Serveral sub state machines are executed for single + slaves. This pointer stores the current slave for these FSMs. +\item[datagram] The FSM class has its own datagram, which is used in + every state and sub-state. +\item[master\_state] This function pointer stores the current state + function for one of the master's state machines. +\item[master\_slaves\_responding] This attribute is used in the + operation state machine (see section~\ref{sec:fsm-op}) to store the + number of slaves, that responded to the last broadcast command. +\item[master\_slave\_states] This attribute stores the slave states, + that were determined by the last broadcast command. +\item[master\_validation] This flag is used by the operation state + machine and is non-zero, if a bus validation has to be done. +\item[slave\_state] This function pointer stores the current state of + the slave scan state machine (see section~\ref{sec:fsm-scan}) or the + slave configuration state machine (see section~\ref{sec:fsm-conf}). +\item[sii\_state] This function pointer stores the current state of + the SII state machine (see section~\ref{sec:fsm-sii}). +\item[sii\_offset] This attribute is used by the SII state machine to + store the word address for the current read or write cycle. +\item[sii\_mode] If this attribute is zero, the SII access is done + with ``auto-increment'' datagrams \cite[section~5.4]{dlspec}. + If it is non-zero, ``station-address'' datagrams are used. +\item[sii\_value] This attribute stores the value to write, or the + read value, respectively. +\item[sii\_start] A timestamp attribute, that stores the beginning + time of an SII operation to detect a timeout. +\item[change\_state] This function pointer stores the current state of + the state change state machine. +\item[change\_new] This attribute stores the requested state for the + state change state machine. +\item[change\_start] A timestamp attribute to detect a timeout while + changing slave states. +\item[coe\_state] This function pointer stores the current state of + the CoE state machines. +\item[sdodata] This is an SDO data object that stores information + about the current SDO to write. +\item[coe\_start] A timestamp attribute to detect timeouts during CoE + configuration. +\end{description} + +\paragraph{Public FSM Methods} + +\begin{description} +\item[ec\_fsm\_init()] Constructor of the FSM class. +\item[ec\_fsm\_clear()] Destructor of the FSM class. +\item[ec\_fsm\_reset()] Resets the whole FSM object. This is needed to + restart the master state machines. +\item[ec\_fsm\_execute()] Executes one state of the current state + machine and then returns. +\item[ec\_fsm\_startup()] Initializes the master startup state + machine, which determines the number of slaves and executes the + slave scan state machine for each slave. +\item[ec\_fsm\_startup\_running()] Returns non-zero, if the startup + state machine did not terminate yet. +\item[ec\_fsm\_startup\_success()] Returns non-zero, if the startup + state machine terminated with success. +\item[ec\_fsm\_configuration()] Initializes the master configuration + state machine, which executes the slave configuration state machine + for each slave. +\item[ec\_fsm\_configuration\_running()] Returns non-zero, if the + configuration state machine did not terminate yet. +\item[ec\_fsm\_configuration\_success()] Returns non-zero, if the + configuration state machine terminated with success. +\end{description} + +\paragraph{FSM State Methods} + +The rest of the methods showed in the UML class diagram are state +methods of the state machines. These states are described in +section~\ref{sec:fsm}. + +%------------------------------------------------------------------------------ + +\subsubsection{The EoE Class} +\label{sec:class-eoe} +\index{EoE!Class} + +Objects of the \textit{ec\_eoe\_t} class are called EoE handlers. Each +EoE handler represents a virtual network interface and can be coupled +to a EoE-capable slave on demand. The UML class diagram can be seen in +figure~\ref{fig:uml-eoe}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.4\textwidth]{images/uml-eoe} + \caption{EoE UML class diagram} + \label{fig:uml-eoe} +\end{figure} + +\paragraph{EoE Attributes} + +\begin{description} +\item[list] The master class maintains a list of EoE handlers. + Therefore this list anchor is needed. +\item[slave] If an EoE handler is coupled to a slave, this pointer + points to the related slave object. Otherwise it is \textit{NULL}. +\item[datagram] Every EoE handler owns a datagram object to exchange + data with the coupled slave via its state machine. +\item[state] This function pointer points to the current state of the + EoE state machine (see section~\ref{sec:eoeimp}). +\item[dev] A pointer to the \textit{net\_device} structure that + represents the network interface to the kernel. +\item[stats] The statistics object for the network interface. +\item[opened] This flag stores, if the network interface was opened. + No EoE processing will be done, if the device is not opened. +\item[t\_last] This timestamp attribute stores the time of the last + bit rate measurement. +\item[rx\_skb] A pointer to the current receive socket buffer. On + every first fragment of a received frame, a new receive socket + buffer is allocated. On every last fragment, this buffer will be + passed to the network stack. +\item[rx\_skb\_offset] This attribute stores the offset for the next + fragment data in the receive socket buffer. +\item[rx\_skb\_size] This attribute stores the current data size of + the receive socket buffer. +\item[rx\_expected\_fragment] The expected number of the next + fragment. If a fragment with an invalid number is received, the + whole frame will be dropped. +\item[rx\_counter] This is the sum of the octets received since the + last bit rate measurement. +\item[rx\_rate] This attribute stores the receive bit rate in bps. +\item[tx\_queue] Each EoE handler maintains a transmittion queue for + frames, that come in via the network interface. This queue is + implemented with a linked list and protected by a spinlock. +\item[tx\_queue\_active] This flag stores, if the transmittion queue + is currently accepting frames from the network stack. If the queue + gets filled up, frame transmittion is suspended with a call to + \textit{netif\_stop\_queue()}. If the fill state decreases below the + half capacity, frame transmittion is restarted with + \textit{netif\_wake\_queue()}. +\item[tx\_queued\_frames] The number of frames in the transmittion + queue. +\item[tx\_queue\_lock] The spinlock used to protect the transmittion + queue. This is needed, because the queue is accessed both from + network stack context and from the master's EoE timer. +\item[tx\_frame] The frame that is currently sent. The + \textit{ec\_eoe\_frame\_t} structure combines the socket buffer + structure with a list head to append it to the transmittion queue. +\item[tx\_frame\_number] The EoE protocol demands to maintain a + sequencial frame number, that must be increased with every frame + sent. +\item[tx\_fragment\_number] The sequencial number of the next fragment + to transmit. +\item[tx\_offset] Current frame data offset for the next fragment to + transmit. +\item[tx\_counter] The number of octets transferred since the last bit + rate measurement. +\item[tx\_rate] The recent transmittion bit rate in bps. +\end{description} + +\paragraph{Public EoE Methods} + +\begin{description} +\item[ec\_eoe\_init()] The EoE handler's constructor. The network + interface is allocated and registered. +\item[ec\_eoe\_clear()] The EoE handler's destructor. The network + interface is unregistered and all allocated memory is freed. +\item[ec\_eoe\_run()] Executes the EoE state machine (see + section~\ref{sec:eoeimp}) for this handler. +\item[ec\_eoe\_active()] Returns true, if the handler has a slave + coupled and the network interface is opened. +\end{description} + +\paragraph{Private EoE Methods} + +\begin{description} +\item[ec\_eoe\_flush()] Clears the transmittion queue and drops all + frames queued for sending. +\item[ec\_eoe\_send()] Sends one fragment of the current frame. +\end{description} + + +\paragraph{EoE State Methods} + +The rest of the private methods are state functions for the EoE state +machine, which is discussed in section~\ref{sec:eoeimp}. + +%------------------------------------------------------------------------------ + +\subsubsection{The Debug Class} +\label{sec:class-debug} + +The debug class maintains a virtual network interface. All frames that +are sent and received by the master will be forwarded to this network +interface, so that bus monitoring can be done with third party tools +(see section~\ref{sec:debug}). Figure~\ref{fig:uml-debug} shows the +UML class diagram. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.3\textwidth]{images/uml-debug} + \caption{Debug UML class diagram} + \label{fig:uml-debug} +\end{figure} + +\paragraph{Debug Attributes} + +\begin{description} +\item[dev] A pointer to the allocated \textit{net\_device} structure + that represents the network interface in the kernel. +\item[stats] An object for interface statistics. +\item[opened] Stores the state of the device. Frames will only be + forwarded, if the device was opened with the \textit{ifconfig up} + command (or something similar). +\end{description} + +\paragraph{Public Debug Methods} + +\begin{description} +\item[ec\_debug\_init()] The constructor. +\item[ec\_debug\_clear()] The destructor. +\item[ec\_debug\_send()] This method forwards a frame to the virtual + network interface. It dynamically allocates a new socket buffer and + passes it to the network stack. +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{The Realtime Interface} +\label{sec:ecrt} +\index{Realtime interface} + +The realtime interface provides functions and data structures for +realtime modules to access and use an EtherCAT master. + +\subsubsection{Master Requesting and Releasing} + +Before a realtime module can access am EtherCAT master provided by the +master module, it has to reserve one for exclusive use. After use, it +has to release the requested master and make it available for other +modules. This is done with the following functions: + +\begin{lstlisting}[language=C] + ec_master_t *ecrt_request_master(unsigned int master_index); + void ecrt_release_master(ec_master_t *master); +\end{lstlisting} + +The \textit{ecrt\_request\_master()} function has to be the first +function a module has to call, when using EtherCAT. The function takes +the index of the master as its argument. The first master has index 0, +the $n$th master has index $n - 1$. The number of existent masters has +to be specified when loading the master module (see +section~\ref{sec:mastermod}). The function tries to reserve the +specified master and scans for slaves. It returns a pointer to the +reserved master object upon success, or \textit{NULL} if an error +occured. + +The \textit{ecrt\_release\_master()} function releases a reserved +master after use. It takes the pointer to the master object returned +by \textit{ecrt\_request\_master()} as its argument and can never +fail. + +\subsubsection{Master Methods} +\label{sec:ecrt-master} + +\paragraph{Domain Creation} + +For process data exchange, at least one process data domain is needed +(see section~\ref{sec:processdata}). + +\begin{lstlisting}[language=C] + ec_domain_t *ecrt_master_create_domain(ec_master_t *master); +\end{lstlisting} + +The \textit{ecrt\_master\_create\_domain()} method creates a new +process data domain and returns a pointer to the new domain object. +This object can be used for registering process data objects and +exchange process data in cyclic operation. On failure, the function +returns \textit{NULL}. + +\paragraph{Slave Handlers} + +To access a certain slave, there is a method to get a slave handler: + +\begin{lstlisting}[language=C] + ec_slave_t *ecrt_master_get_slave(const ec_master_t *, + const char *); +\end{lstlisting} + +The \textit{ecrt\_master\_get\_slave()} method returns a pointer to a +certain slave object, specified by its ASCII address (see +section~\ref{sec:addr}). If the address is invalid, \textit{NULL} is +returned. + +\paragraph{Master Activation} + +When all domains are created, and all process data objects are +registered, the master can be activated: + +\begin{lstlisting}[language=C] + int ecrt_master_activate(ec_master_t *master); + void ecrt_master_deactivate(ec_master_t *master); +\end{lstlisting} + +By calling the \textit{ecrt\_master\_activate()} method, all slaves +are configured according to the prior method calls and are brought +into \textit{OP} state. In this case, the method has a return value of +0. Otherwise (wrong configuration or bus failure) the method returns +non-zero. + +The \textit{ecrt\_master\_deactivate()} method is the counterpart to +the activate call: It brings all slaves back into \textit{INIT} state +again. This method should be called prior to +\textit{ecrt\_\-master\_\-release()}. + +\paragraph{Locking Callbacks} + +For concurrent master access, the realtime module has to provide a +locking mechanism (see section~\ref{sec:concurr}): + +\begin{lstlisting}[language=C] + void ecrt_master_callbacks(ec_master_t *master, + int (*request_cb)(void *), + void (*release_cb)(void *), + void *cb_data); +\end{lstlisting} + +The ``request lock'' and ``release lock'' callbacks can be set with +the \textit{ecrt\_master\_call\-backs()} method. It takes two function +pointers and a data value as additional arguments. The arbitrary data +value will be passed as argument on every callback. Asynchronous +master access (like EoE processing) is only possible if these +callbacks have been set. + +\paragraph{Preparation of Cyclic Data Exchange} + +Cyclic operation mostly consists of the three steps input, processing +and output. In EtherCAT terms this would mean: Receive datagrams, +evaluate process data and send datagrams. The first cycle differs from +this principle, because no datagrams have been sent yet, so there is +nothing to receive. To avoid having a case differantiation (in terms +of an \textit{if} clause), the following method exists: + +\begin{lstlisting}[language=C] + void ecrt_master_prepare(ec_master_t *master); +\end{lstlisting} + +As a last thing before cyclic operation, a call to the +\textit{ecrt\_master\_prepare()} method should be issued. It makes all +process data domains queue their datagrams and issues a send command, +so that the first receive call in cyclic operation will not fail. + +\paragraph{Frame Sending and Receiving} + +To send all queued datagrams and to later receive the sent datagrams +there are two methods: + +\begin{lstlisting}[language=C] + void ecrt_master_send(ec_master_t *master); + void ecrt_master_receive(ec_master_t *master); +\end{lstlisting} + +The \textit{ecrt\_master\_send()} method takes all datagrams, that +have been queued for transmission, packs them into frames, and passes +them to the network device for sending. + +The \textit{ecrt\_master\_receive()} queries the network device for +received frames (by calling the ISR\index{ISR}), extracts received +datagrams and dispatches the results to the datagram objects in the +queue. Received datagrams, and the ones that timed out, will be +marked, and then dequeued. + +\paragraph{Running the Operation State Machine} + +The master's operation state machine (see section~\ref{sec:fsm-op}) +monitors the bus in cyclic operation and reconfigures slaves, if +necessary. Therefore, the following method should be called +cyclically: + +\begin{lstlisting}[language=C] + void ecrt_master_run(ec_master_t *master); +\end{lstlisting} + +The \textit{ecrt\_master\_run()} method executes the master's +operation state machine step by step. It returns after processing one +state and queuing a datagram. Calling this function is not mandatory, +but highly recommended. + +\paragraph{Master Monitoring} + +It is also highly recommended to evaluate the master's error state. In +this way it is possible to notice lost network links, failed bus +segments, and other issues: + +\begin{lstlisting}[language=C] + int ecrt_master_state(const ec_master_t *master); +\end{lstlisting} + +The \textit{ecrt\_master\_state()} method returns the master's error +state. The following states are defined as part of the realtime +interface: + +\begin{description} +\item[EC\_MASTER\_OK] means, that no error has occurred. +\item[EC\_MASTER\_LINK\_ERROR] means, that the network link is + currently down. +\item[EC\_MASTER\_BUS\_ERROR] means, that one or more slaves do not + respond. +\end{description} + +\subsubsection{Domain Methods} +\label{sec:ecrt-domain} + +\paragraph{PDO Registration} + +To access data of a slave's PDO in cyclic operation, it is necessary +to make it part of a process data domain: + +\begin{lstlisting}[language=C] + ec_slave_t *ecrt_domain_register_pdo(ec_domain_t *domain, + const char *address, + uint32_t vendor_id, + uint32_t product_code, + const char *pdo_name + void **data_ptr); + int ecrt_domain_register_pdo_list(ec_domain_t *domain, + const ec_pdo_reg_t *pdos); +\end{lstlisting} + +The \textit{ecrt\_domain\_register\_pdo()} method registers a certain +PDO as part of the domain and takes the address of the process data +pointer. This pointer will be set on master activation and then can be +parameter to the \textit{EC\_READ\_*} and \textit{EC\_WRITE\_*} macros +described below. + +A perhaps easier way to register multiple PDOs at the same time is to +fill an array of \textit{ec\_pdo\_reg\_t} and hand it to the +\textit{ecrt\_domain\_register\_pdo\_list()} method. Attention: This +array has to be terminated by an empty structure (\textit{\{\}})! + +\paragraph{Evaluating Domain Data} + +To evaluate domain data, the following method has to be used: + +\begin{lstlisting}[language=C] + void ecrt_domain_process(ec_domain_t *domain); +\end{lstlisting} + +The \textit{ecrt\_domain\_process()} method sets the domains state and +requeues its datagram for sending. + +\paragraph{Domain State} + +Similar to the master state, a domain has an own error state: + +\begin{lstlisting}[language=C] + int ecrt_domain_state(const ec_domain_t *domain); +\end{lstlisting} + +The \textit{ecrt\_domain\_state()} method returns the domain's error +state. It is non-zero if \underline{not} all process data values could +be exchanged, and zero otherwise. + +\subsubsection{Slave Methods} +\label{sec:ecrt-slave} + +\paragraph{SDO Configuration} + +To configure slave SDOs, the function interface below can be used: + +\begin{lstlisting}[language=C] + int ecrt_slave_conf_sdo8(ec_slave_t *slave, + uint16_t sdo_index, + uint8_t sdo_subindex, + uint8_t value); + int ecrt_slave_conf_sdo16(ec_slave_t *slave, + uint16_t sdo_index, + uint8_t sdo_subindex, + uint16_t value); + int ecrt_slave_conf_sdo32(ec_slave_t *slave, + uint16_t sdo_index, + uint8_t sdo_subindex, + uint32_t value); +\end{lstlisting} + +The \textit{ecrt\_slave\_conf\_sdo*()} methods prepare the +configuration of a certain SDO. The index and subindex of the SDO, and +the value have to be specified. The configuration is done each time, +the slave is reconfigured. The methods only differ in the SDO's data +type. If the configuration could be prepared, zero is returned. If an +error occured, non-zero is returned. + +\paragraph{Variable-sized PDOs} + +For specifying the size of variable-sized PDOs, the following method +can be used: + +\begin{lstlisting}[language=C] + int ecrt_slave_pdo_size(ec_slave_t *slave, + const char *pdo_name, + size_t size); +\end{lstlisting} + +The \textit{ecrt\_slave\_pdo\_size()} method takes the name of the PDO +and the size. It returns zero on success, otherwise non-zero. + +\subsubsection{Process Data Access} +\label{sec:macros} + +The endianess of the process data could differ from that of the CPU. +Therefore, process data access has to be done by the macros below, +that are also provided by the realtime interface: + +\begin{lstlisting}[language=C] + #define EC_READ_BIT(DATA, POS) + #define EC_WRITE_BIT(DATA, POS, VAL) + + #define EC_READ_U8(DATA) + #define EC_READ_S8(DATA) + #define EC_READ_U16(DATA) + #define EC_READ_S16(DATA) + #define EC_READ_U32(DATA) + #define EC_READ_S32(DATA) + + #define EC_WRITE_U8(DATA, VAL) + #define EC_WRITE_S8(DATA, VAL) + #define EC_WRITE_U16(DATA, VAL) + #define EC_WRITE_S16(DATA, VAL) + #define EC_WRITE_U32(DATA, VAL) + #define EC_WRITE_S32(DATA, VAL) +\end{lstlisting} + +There are macros for bitwise access (\textit{EC\_READ\_BIT()}, +\textit{EC\_WRITE\_BIT()}), and bytewise access +(\textit{EC\_READ\_*()}, \textit{EC\_WRITE\_*()}). The bytewise macros +carry the data type in their name. Example: \textit{EC\_WRITE\_S16()} +writes a 16 bit signed value to EtherCAT data. The \textit{DATA} +parameter is supposed to be a process data pointer, as provided at PDO +registration. + +The macros use the kernel's endianess conversion macros, that are +preprocessed to empty macros in case of equal endianess. This is the +definition for the \textit{EC\_\-READ\_\-U16()} macro: + +\begin{lstlisting}[language=C] + #define EC_READ_U16(DATA) \ + ((uint16_t) le16_to_cpup((void *) (DATA))) +\end{lstlisting} + +The \textit{le16\_to\_cpup()} macro converts a little-endian, 16 bit +value to the CPU's architecture and takes a pointer to the input value +as its argument. If the CPU's architecture is little-endian, too (for +example on X86 and compatible), nothing has to be converted. In this +case, the macro is replaced with an empty macro by the preprocessor +and so there is no unneeded function call or case differentiation in +the code. + +For keeping it portable, it is highly recommended to make use of these +macros. + +%------------------------------------------------------------------------------ + +\subsection{Slave Addressing} +\label{sec:addr} +\index{Slave!Addressing} + +The master offers the serveral slave addressing schemes (for PDO +registration or configuration) via the realtime interface. For this +reason, slave addresses are ASCII\nomenclature{ASCII}{American + Standard Code for Information Interchange}-coded and passed as a +string. The addressing schemes are independent of the EtherCAT +protocol and represent an additional feature of the master. + +Below, the allowed addressing schemes are described. The descriptions +are followed by a regular expression formally defining the addressing +scheme, and one or more examples. + +\begin{description} +\item[Position Addressing] This is the normal addressing scheme, where + each slave is addressed by its ring position. The first slave has + address 0, and the $n$th slave has address $n - 1$. This addressing + scheme is useful for small busses that have a fixed number of slaves.\\ + RegEx: \texttt{[0-9]+} --- Example: \texttt{"42"} +\item[Advanced Position Addressing] Bus couplers segment the bus into + (physical) blocks. Though the logical ring positions keep being the + same, it is easier to address a slave with its block number and the + relative position inside the block. This addressing is done by + passing the (zero-based) index of the bus coupler (not the coupler's + ring position), followed by a colon and the relative position of the + actual slave starting at the bus coupler.\\ + RegEx: \texttt{[0-9]+:[0-9]+} --- Examples: \texttt{"0:42"}, + \texttt{"2:7"} +\item[Alias Addressing] Each slave can have a ``secondary slave + address'' or ``alias address''\footnote{Information about how to set + the alias can be found in section~\ref{sec:eepromaccess}} stored + in its E$^2$PROM. The alias is evaluated by the master and can be + used to address the slave, which is useful when a clearly defined + slave has to be addressed and the ring position is not known or can + change over time. This scheme is used by starting the address string + with a mesh (\#) followed by the alias address. The latter can also + be provided as hexadecimal value, prefixed with \textit{0x}.\\ + RegEx: \texttt{\#(0x[0-9A-F]+|[0-9]+)} --- Examples: + \texttt{"\#6622"}, \texttt{"\#0xBEEF"} +\item[Advanced Alias Addressing] This is a mixture of the ``Alias + Addressing'' and ``Advanced Position Addressing'' schemes. A certain + slave is addressed by specifying its relative position after an + aliased slave. This is very useful, if a complete block of slaves + can vary its position in the bus. The bus coupler preceeding the + block should get an alias. The block slaves can then be addressed by + specifying this alias and their position inside the block. This + scheme is used by starting the address string with a mesh (\#) + followed by the alias address (which can be hexadecimal), then a + colon and the relative posision of the slave to + address.\\ + RegEx: \texttt{\#(0x[0-9A-F]+|[0-9]+):[0-9]+} --- Examples: + \texttt{"\#0xBEEF:7"}, \texttt{"\#6:2"} +\end{description} + +In anticipation of section~\ref{sec:ecrt}, the functions accepting +these address strings are \textit{ecrt\_\-master\_\-get\_slave()}, +\textit{ecrt\_domain\_register\_pdo()} and +\textit{ecrt\_domain\_register\_pdo\_list()} (the latter through the +\textit{ec\_pdo\_reg\_t} structure). + +%------------------------------------------------------------------------------ + +\subsection{Concurrent Master Access} +\label{sec:concurr} +\index{Concurrency} + +In some cases, one master is used by serveral instances, for example +when a realtime module does cyclic process data exchange, and there +are EoE-capable slaves that require to exchange Ethernet data with the +kernel (see section~\ref{sec:eoeimp}). For this reason, the master is +a shared resource, and access to it has to be sequenctialized. This is +usually done by locking with semaphores, or other methods to protect +critical sections. + +The master itself can not provide locking mechanisms, because it has +no chance to know the appropriate kind of lock. Imagine, the realtime +module uses RTAI functionality, then ordinary kernel semaphores would +not be sufficient. For that, an important design decision was made: +The realtime module that reserved a master must have the total +control, therefore it has to take responsibility for providing the +appropriate locking mechanisms. If another instance wants to access +the master, it has to request the master lock by callbacks, that have +to be set by the realtime module. Moreover the realtime module can +deny access to the master if it consideres it to be awkward at the +moment. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.6\textwidth]{images/master-locks} + \caption{Concurrent master access} + \label{fig:locks} +\end{figure} + +Figure~\ref{fig:locks} exemplary shows, how two processes share one +master: The realtime module's cyclic thread uses the master for +process data exchange, while the master-internal EoE process uses it +to communicate with EoE-capable slaves. Both have to aquire the master +lock before access: The realtime thread can access the lock natively, +while the EoE process has to use the master callbacks. +Section~\ref{sec:concurrency} gives an example, of how to implement +this. + +%------------------------------------------------------------------------------ + +\section{The Master's State Machines} +\label{sec:fsm} +\index{FSM} + +Many parts of the EtherCAT master are implemented as \textit{finite + state machines} (FSMs\nomenclature{FSM}{Finite State Machine}). +Though this leads to a higher grade of complexity in some aspects, is +opens many new possibilities. + +The below short code example exemplary shows how to read all slave +states and moreover illustrates the restrictions of ``sequential'' +coding: + +\begin{lstlisting}[language=C,numbers=left] + ec_datagram_brd(datagram, 0x0130, 2); // prepare datagram + if (ec_master_simple_io(master, datagram)) return -1; + slave_states = EC_READ_U8(datagram->data); // process datagram +\end{lstlisting} + +The \textit{ec\_master\_simple\_io()} function provides a simple +interface for synchronously sending a single datagram and receiving +the result\footnote{For all communication issues have been meanwhile + sourced out into state machines, the function is deprecated and + stopped existing. Nevertheless it is adequate for showing it's own + restrictions.}. Internally, it queues the specified datagram, +invokes the \textit{ec\_master\_send\_datagrams()} function to send a +frame with the queued datagram and then waits actively for its +reception. + +This sequential approach is very simple, reflecting in only three +lines of code. The disadvantage is, that the master is blocked for the +time it waits for datagram reception. There is no difficulty when only +one instance is using the master, but if more instances want to +(synchronously\footnote{At this time, synchronous master access will + be adequate to show the advantages of an FSM. The asynchronous + approach will be discussed in section~\ref{sec:eoeimp}}) use the +master, it is inevitable to think about an alternative to the +sequential model. + +Master access has to be sequentialized for more than one instance +wanting to send and receive datagrams synchronously. With the present +approach, this would result in having one phase of active waiting for +each instance, which would be non-acceptable especially in realtime +circumstances, because of the huge time overhead. + +A possible solution is, that all instances would be executed +sequentially to queue their datagrams, then give the control to the +next instance instead of waiting for the datagram reception. Finally, +bus IO is done by a higher instance, which means that all queued +datagrams are sent and received. The next step is to execute all +instances again, which then process their received datagrams and issue +new ones. + +This approach results in all instances having to retain their state, +when giving the control back to the higher instance. It is quite +obvious to use a \textit{finite state machine} model in this case. +Section~\ref{sec:fsmtheory} will introduce some of the theory used, +while the listings below show the basic approach by coding the example +from above as a state machine: + +\begin{lstlisting}[language=C,numbers=left] + // state 1 + ec_datagram_brd(datagram, 0x0130, 2); // prepare datagram + ec_master_queue(master, datagram); // queue datagram + next_state = state_2; + // state processing finished +\end{lstlisting} + +After all instances executed their current state and queued their +datagrams, these are sent and received. Then the respective next +states are executed: + +\begin{lstlisting}[language=C,numbers=left] + // state 2 + if (datagram->state != EC_DGRAM_STATE_RECEIVED) { + next_state = state_error; + return; // state processing finished + } + slave_states = EC_READ_U8(datagram->data); // process datagram + // state processing finished. +\end{lstlisting} + +See section~\ref{sec:statemodel} for an introduction to the +state machine programming concept used in the master code. + +%------------------------------------------------------------------------------ + +\subsection{State Machine Theory} +\label{sec:fsmtheory} +\index{FSM!Theory} + +A finite state machine \cite{automata} is a model of behavior with +inputs and outputs, where the outputs not only depend on the inputs, +but the history of inputs. The mathematical definition of a finite +state machine (or finite automaton) is a six-tuple $(\Sigma, \Gamma, +S, s_0, \delta, \omega)$, with + +\begin{itemize} +\item the input alphabet $\Sigma$, with $\Sigma \neq + \emptyset$, containing all input symbols, +\item the output alphabet $\Gamma$, with $\Gamma \neq + \emptyset$, containing all output symbols, +\item the set of states $S$, with $S \neq \emptyset$, +\item the set of initial states $s_0$ with $s_0 \subseteq S, s_0 \neq + \emptyset$ +\item the transition function $\delta: S \times \Sigma \rightarrow S + \times \Gamma$ +\item the output function $\omega$. +\end{itemize} + +The state transition function $\delta$ is often specified by a +\textit{state transition table}, or by a \textit{state transition + diagram}. The transition table offers a matrix view of the state +machine behavior (see table~\ref{tab:statetrans}). The matrix rows +correspond to the states ($S = \{s_0, s_1, s_2\}$) and the columns +correspond to the input symbols ($\Gamma = \{a, b, \varepsilon\}$). +The table contents in a certain row $i$ and column $j$ then represent +the next state (and possibly the output) for the case, that a certain +input symbol $\sigma_j$ is read in the state $s_i$. + +\begin{table}[htbp] + \caption{A typical state transition table} + \label{tab:statetrans} + \vspace{2mm} + \centering + \begin{tabular}{l|ccc} + & $a$ & $b$ & $\varepsilon$\\ \hline + $s_0$ & $s_1$ & $s_1$ & $s_2$\\ + $s_1$ & $s_2$ & $s_1$ & $s_0$\\ + $s_2$ & $s_0$ & $s_0$ & $s_0$\\ \hline + \end{tabular} +\end{table} + +The state diagram for the same example looks like the one in +figure~\ref{fig:statetrans}. The states are represented as circles or +ellipses and the transitions are drawn as arrows between them. Close +to a transition arrow can be the condition that must be fulfilled to +allow the transition. The initial state is marked by a filled black +circle with an arrow pointing to the respective state. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.5\textwidth]{images/statetrans} + \caption{A typical state transition diagram} + \label{fig:statetrans} +\end{figure} + +\paragraph{Deterministic and non-deterministic state machines} + +A state machine can be deterministic, meaning that for one state and +input, there is one (and only one) following state. In this case, the +state machine has exactly one starting state. Non-deterministic state +machines can have more than one transitions for a single state-input +combination. There is a set of starting states in the latter case. + +\paragraph{Moore and Mealy machines} + +There is a distinction between so-called \textit{Moore machines}, and +\textit{Mealy machines}. Mathematically spoken, the distinction lies +in the output function $\omega$: If it only depends on the current +state ($\omega: S \rightarrow \Gamma$), the machine corresponds to the +``Moore Model''. Otherwise, if $\omega$ is a function of a state and +the input alphabet ($\omega: S \times \Sigma \rightarrow \Gamma$) the +state machine corresponds to the ``Mealy model''. Mealy machines are +the more practical solution in most cases, because their design allows +machines with a minimum number of states. In practice, a mixture of +both models is often used. + +\paragraph{Misunderstandings about state machines} + +There is a phenomenon called ``state explosion'', that is oftenly +taken as a counter-argument against general use of state machines in +complex environments. It has to be mentioned, that this point is +misleading~\cite{fsmmis}. State explosions happen usually as a result +of a bad state machine design: Common mistakes are storing the present +values of all inputs in a state, or not dividing a complex state +machine into simpler sub state machines. The EtherCAT master uses +serveral state machines, that are executed hierarchically and so serve +as sub state machines. These are also described below. + +%------------------------------------------------------------------------------ + +\subsection{The Master's State Model} +\label{sec:statemodel} + +This section will introduce the techniques used in the master to +implement state machines. + +\paragraph{State Machine Programming} + +There are certain ways to implement a state machine in \textit{C} +code. An obvious way is to implement the different states and actions +by one big case differentiation: + +\begin{lstlisting}[language=C,numbers=left] + enum {STATE_1, STATE_2, STATE_3}; + int state = STATE_1; + + void state_machine_run(void *priv_data) { + switch (state) { + case STATE_1: + action_1(); + state = STATE_2; + break; + case STATE_2: + action_2() + if (some_condition) state = STATE_1; + else state = STATE_3; + break; + case STATE_3: + action_3(); + state = STATE_1; + break; + } + } +\end{lstlisting} + +For small state machines, this is an option. The disadvantage is, that +with an increasing number of states the code soon gets complex and an +additional case differentiation is executed each run. Besides, lots of +indentation is wasted. + +The method used in the master is to implement every state in an own +function and to store the current state function with a function +pointer: + +\begin{lstlisting}[language=C,numbers=left] + void (*state)(void *) = state1; + + void state_machine_run(void *priv_data) { + state(priv_data); + } + + void state1(void *priv_data) { + action_1(); + state = state2; + } + + void state2(void *priv_data) { + action_2(); + if (some_condition) state = state1; + else state = state2; + } + + void state3(void *priv_data) { + action_3(); + state = state1; + } +\end{lstlisting} + +In the master code, state pointers of all state machines\footnote{All + except for the EoE state machine, because multiple EoE slaves have + to be handled in parallel. For this reason each EoE handler object + has its own state pointer.} are gathered in a single object of the +\textit{ec\_fsm\_t} class. This is advantageous, because there is +always one instance of every state machine available and can be +started on demand. + +\paragraph{Mealy and Moore} + +If a closer look is taken to the above listing, it can be seen that +the actions executed (the ``outputs'' of the state machine) only +depend on the current state. This accords to the ``Moore'' model +introduced in section~\ref{sec:fsmtheory}. As mentioned, the ``Mealy'' +model offers a higher flexibility, which can be seen in the listing +below: + +\begin{lstlisting}[language=C,numbers=left] + void state7(void *priv_data) { + if (some_condition) { + action_7a(); + state = state1; + } + else { + action_7b(); + state = state8; + } + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3} + \textcircled{\tiny 7}] The + state function executes the actions depending on the state + transition, that is about to be done. +\end{description} + +The most flexible alternative is to execute certain actions depending +on the state, followed by some actions dependent on the state +transition: + +\begin{lstlisting}[language=C,numbers=left] + void state9(void *priv_data) { + action_9(); + if (some_condition) { + action_9a(); + state = state7; + } + else { + action_9b(); + state = state10; + } + } +\end{lstlisting} + +This model is oftenly used in the master. It combines the best aspects +of both approaches. + +\paragraph{Using Sub State Machines} + +To avoid having too much states, certain functions of the EtherCAT +master state machine have been sourced out into sub state machines. +This helps to encapsule the related workflows and moreover avoids the +``state explosion'' phenomenon described in +section~\ref{sec:fsmtheory}. If the master would instead use one big +state machine, the number of states would be a multiple of the actual +number. This would increase the level of complexity to a +non-manageable grade. + +\paragraph{Executing Sub State Machines} + +If a state machine starts to execute a sub state machine, it usually +remains in one state until the sub state machine terminates. This is +usually done like in the listing below, which is taken out of the +slave configuration state machine code: + +\begin{lstlisting}[language=C,numbers=left] + void ec_fsm_slaveconf_saveop(ec_fsm_t *fsm) + { + fsm->change_state(fsm); // execute state change + // sub state machine + + if (fsm->change_state == ec_fsm_error) { + fsm->slave_state = ec_fsm_end; + return; + } + + if (fsm->change_state != ec_fsm_end) return; + + // continue state processing + ... +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] \textit{change\_state} is the + state pointer of the state change state machine. The state function, + the pointer points on, is executed\ldots +\item[\normalfont\textcircled{\tiny 6}] \ldots either until the state + machine terminates with the error state \ldots +\item[\normalfont\textcircled{\tiny 11}] \ldots or until the state + machine terminates in the end state. Until then, the ``higher'' + state machine remains in the current state and executes the sub + state machine again in the next cycle. +\end{description} + +\paragraph{State Machine Descriptions} + +The below sections describe every state machine used in the EtherCAT +master. The textual descriptions of the state machines contain +references to the transitions in the corresponding state transition +diagrams, that are marked with an arrow followed by the name of the +successive state. Transitions caused by trivial error cases (i.~e. no +response from slave) are not described explicitly. These transitions +are drawn as dashed arrows in the diagrams. + +%------------------------------------------------------------------------------ + +\subsection{The Operation State Machine} +\label{sec:fsm-op} +\index{FSM!Operation} + +The Operation state machine is executed by calling the +\textit{ecrt\_master\_run()} method in cyclic realtime code. Its +purpose is to monitor the bus and to reconfigure slaves after a bus +failure or power failure. Figure~\ref{fig:fsm-op} shows its transition +diagram. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.8\textwidth]{images/fsm-op} + \caption{Transition diagram of the operation state machine} + \label{fig:fsm-op} +\end{figure} + +\begin{description} +\item[START] This is the beginning state of the operation state + machine. There is a datagram issued, that queries the ``AL Control + Response'' attribute \cite[section~5.3.2]{alspec} of all slaves via + broadcast. In this way, all slave states and the number of slaves + responding can be determined. $\rightarrow$~\textit{BROADCAST} + +\item[BROADCAST] The broadcast datagram is evaluated. A change in the + number of responding slaves is treates as a topology change. If the + number of slaves is not as expected, the bus is marked as + ``tainted''. In this state, no slave reconfiguration is possible, + because the assignment of known slaves and those present on the bus + is ambiguous. If the number of slaves is considered as right, the + bus is marked for validation, because it turned from tainted to + normal state and it has to be checked, if all slaves are valid. Now, + the state of every single slave has to be determined. For that, a + (unicast) datagram is issued, that queries the first slave's ``AL + Control Response'' attribute. $\rightarrow$~\textit{READ STATES} + +\item[READ STATES] If the current slave did not respond to its + configured station address, it is marked as offline, and the next + slave is queried. $\rightarrow$~\textit{READ STATES} + + If the slave responded, it is marked as online and its current state + is stored. The next slave is queried. $\rightarrow$~\textit{READ + STATES} + + If all slaves have been queried, and the bus is marked for + validation, the validation is started by checking the first slaves + vendor ID. $\rightarrow$~\textit{VALIDATE VENDOR} + + If no validation has to be done, it is checked, if all slaves are in + the state they are supposed to be. If not, the first of slave with + the wrong state is reconfigured and brought in the required state. + $\rightarrow$~\textit{CONFIGURE SLAVES} + + If all slaves are in the correct state, the state machine is + restarted. $\rightarrow$~\textit{START} + +\item[CONFIGURE SLAVES] The slave configuration state machine is + executed until termination. $\rightarrow$~\textit{CONFIGURE SLAVES} + + If there are still slaves in the wrong state after another check, + the first of these slaves is configured and brought into the correct + state again. $\rightarrow$~\textit{CONFIGURE SLAVES} + + If all slaves are in the correct state, the state machine is + restarted. $\rightarrow$~\textit{START} + +\item[VALIDATE VENDOR] The SII state machine is executed until + termination. If the slave has the wrong vendor ID, the state machine + is restarted. $\rightarrow$~\textit{START} + + If the slave has the correct vendor ID, its product ID is queried. + $\rightarrow$~\textit{VALIDATE PRODUCT} + +\item[VALIDATE PRODUCT] The SII state machine is executed until + termination. If the slave has the wrong product ID, the state + machine is restarted. $\rightarrow$~\textit{START} + + If the slave has the correct product ID, the next slave's vendor ID + is queried. $\rightarrow$~\textit{VALIDATE VENDOR} + + If all slaves have the correct vendor IDs and product codes, the + configured station addresses can be safely rewritten. This is done + for the first slave marked as offline. + $\rightarrow$~\textit{REWRITE ADDRESSES} + +\item[REWRITE ADDRESSES] If the station address was successfully + written, it is sear\-ched for the next slave marked as offline. If + there is one, its address is reconfigured, too. + $\rightarrow$~\textit{REWRITE ADDRESSES} + + If there are no more slaves marked as offline, the state machine is + restarted. $\rightarrow$~\textit{START} +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{The Idle State Machine} +\label{sec:fsm-idle} +\index{FSM!Idle} + +The Idle state machine is executed by a kernel workqueue, if no +realtime module is connected. Its purpose is to make slave information +available to user space, operate EoE-capable slaves, read and write +E$^2$PROM contents and test slave functionality. +Figure~\ref{fig:fsm-idle} shows its transition diagram. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.8\textwidth]{images/fsm-idle} + \caption{Transition diagram of the idle state machine} + \label{fig:fsm-idle} +\end{figure} + +\begin{description} +\item[START] The beginning state of the idle state machine. Similar to + the operation state machine, a broadcast datagram is issued, to + query all slave states and the number of slaves. + $\rightarrow$~\textit{BROADCAST} + +\item[BROADCAST] The number of responding slaves is evaluated. If it + has changed since the last time, this is treated as a topology + change and the internal list of slaves is cleared and rebuild + completely. The slave scan state machine is started for the first + slave. $\rightarrow$~\textit{SCAN FOR SLAVES} + + If no topology change happened, every single slave state is fetched. + $\rightarrow$~\textit{READ STATES} + +\item[SCAN FOR SLAVES] The slave scan state machine is executed until + termination. $\rightarrow$~\textit{SCAN FOR SLAVES} + + If there is another slave to scan, the slave scan state machine is + started again. $\rightarrow$~\textit{SCAN FOR SLAVES} + + If all slave information has been fetched, slave addresses are + calculated and EoE processing is started. Then, the state machine is + restarted. $\rightarrow$~\textit{START} + +\item[READ STATES] If the slave did not respond to the query, it is + marked as offline. The next slave is queried. + $\rightarrow$~\textit{READ STATES} + + If the slave responded, it is marked as online. And the next slave + is queried. $\rightarrow$~\textit{READ STATES} + + If all slave states have been determined, it is checked, if any + slaves are not in the state they supposed to be. If this is true, + the slave configuration state machine is started for the first of + them. $\rightarrow$~\textit{CONFIGURE SLAVES} + + If all slaves are in the correct state, it is checked, if any + E$^2$PROM write operations are pending. If this is true, the first + pending operation is executed by starting the SII state machine for + writing access. $\rightarrow$~\textit{WRITE EEPROM} + + If all these conditions are false, there is nothing to do and the + state machine is restarted. $\rightarrow$~\textit{START} + +\item[CONFIGURE SLAVES] The slave configuration state machine is + executed until termination. $\rightarrow$~\textit{CONFIGURE SLAVES} + + After this, it is checked, if another slave needs a state change. If + this is true, the slave state change state machine is started for + this slave. $\rightarrow$~\textit{CONFIGURE SLAVES} + + If all slaves are in the correct state, it is determined, if any + E$^2$PROM write operations are pending. If this is true, the first + pending operation is executed by starting the SII state machine for + writing access. $\rightarrow$~\textit{WRITE EEPROM} + + If all prior conditions are false, the state machine is restarted. + $\rightarrow$~\textit{START} + +\item[WRITE EEPROM] The SII state machine is executed until + termination. $\rightarrow$~\textit{WRITE EEPROM} + + If the current word has been written successfully, and there are + still word to write, the SII state machine is started for the next + word. $\rightarrow$~\textit{WRITE EEPROM} + + If all words have been written successfully, the new E$^2$PROM + contents are evaluated and the state machine is restarted. + $\rightarrow$~\textit{START} + +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{The Slave Scan State Machine} +\label{sec:fsm-scan} +\index{FSM!Slave Scan} + +The slave scan state machine, which can be seen in +figure~\ref{fig:fsm-slavescan}, leads through the process of fetching +all slave information. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.6\textwidth]{images/fsm-slavescan} + \caption{Transition diagram of the slave scan state machine} + \label{fig:fsm-slavescan} +\end{figure} + +\begin{description} +\item[START] In the beginning state of the slave scan state machine, + the station address is written to the slave, which is always the + ring position~+~$1$. In this way, the address 0x0000 (default + address) is not used, which makes it easy to detect unconfigured + slaves. $\rightarrow$~\textit{ADDRESS} + +\item[ADDRESS] The writing of the station address is verified. After + that, the slave's ``AL Control Response'' attribute is queried. + $\rightarrow$~\textit{STATE} + +\item[STATE] The AL state is evaluated. A warning is output, if the + slave has still the \textit{Change} bit set. After that, the slave's + ``DL Information'' attribute is queried. + $\rightarrow$~\textit{BASE} + +\item[BASE] The queried base data are evaluated: Slave type, revision + and build number, and even more important, the number of supported + sync managers and FMMUs are stored. After that, the slave's data + link layer information is read from the ``DL Status'' attribute at + address 0x0110. $\rightarrow$~\textit{DATALINK} + +\item[DATALINK] In this state, the DL information is evaluated: This + information about the communication ports contains, if the link is + up, if the loop has been closed and if there is a carrier detected + on the RX side of each port. + + Then, the state machine starts measuring the size of the slave's + E$^2$PROM contents. This is done by subsequently reading out each + category header, until the last category is reached (type 0xFFFF). + This procedure is started by querying the first category header at + word address 0x0040 via the SII state machine. + $\rightarrow$~\textit{EEPROM SIZE} + +\item[EEPROM SIZE] The SII state machine is executed until + termination. $\rightarrow$~\textit{EEPROM SIZE} + + If the category type does not mark the end of the categories, the + position of the next category header is determined via the length of + the current category, and the SII state machine is started again. + $\rightarrow$~\textit{EEPROM SIZE} + + If the size of the E$^2$PROM contents has been determined, memory is + allocated, to read all the contents. The SII state machine is + started to read the first word. $\rightarrow$~\textit{EEPROM DATA} + +\item[EEPROM DATA] The SII state machine is executed until + termination. $\rightarrow$~\textit{EEPROM DATA} + + Two words have been read. If more than one word is needed, the two + words are written in the allocated memory. Otherwise only one word + (the last word) is copied. If more words are to read, the SII state + machine is started again to read the next two words. + $\rightarrow$~\textit{EEPROM DATA} + + The complete E$^2$PROM contents have been read. The slave's identity + object and mailbox information are evaluated. Moreover the category + types STRINGS, GENERAL, SYNC and PDO are evaluated. The slave + scanning has been completed. $\rightarrow$~\textit{END} + +\item[END] Slave scanning has been finished. + +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{The Slave Configuration State Machine} +\label{sec:fsm-conf} +\index{FSM!Slave Configuration} + +The slave configuration state machine, which can be seen in +figure~\ref{fig:fsm-slaveconf}, leads through the process of +configuring a slave and bringing it to a certain state. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.6\textwidth]{images/fsm-slaveconf} + \caption{Transition diagram of the slave configuration state + machine} + \label{fig:fsm-slaveconf} +\end{figure} + +\begin{description} +\item[INIT] The state change state machine has been initialized to + bring the slave into the \textit{INIT} state. Now, the slave state + change state machine is executed until termination. + $\rightarrow$~\textit{INIT} + + If the slave state change failed, the configuration has to be + aborted. $\rightarrow$~\textit{END} + + The slave state change succeeded and the slave is now in + \textit{INIT} state. If this is the target state, the configuration + is finished. $\rightarrow$~\textit{END} + + If the slave does not support any sync managers, the sync manager + configuration can be skipped. The state change state machine is + started to bring the slave into \textit{PREOP} state. + $\rightarrow$~\textit{PREOP} + + Sync managers are configured conforming to the sync manager category + information provided in the slave's E$^2$PROM. The corresponding + datagram is issued. $\rightarrow$~\textit{SYNC} + +\item[SYNC] If the sync manager configuration datagram is accepted, + the sync manager configuration was successful. The slave may now + enter the \textit{PREOP} state, and the state change state machine + is started. $\rightarrow$~\textit{PREOP} + +\item[PREOP] The state change state machine is executed until + termination. $\rightarrow$~\textit{PREOP} + + If the state change failed, the configuration has to be aborted. + $\rightarrow$~\textit{END} + + If the \textit{PREOP} state was the target state, the configuration + is finished. $\rightarrow$~\textit{END} + + If the slave supports no FMMUs, the FMMU configuration can be + skipped. If the slave has SDOs to configure, it is begun with + sending the first SDO. $\rightarrow$~\textit{SDO\_CONF} + + If no SDO configurations are provided, the slave can now directly be + brought into the \textit{SAVEOP} state and the state change state + machine is started again. $\rightarrow$~\textit{SAVEOP} + + Otherwise, all supported FMMUs are configured according to the PDOs + requested via the master's realtime interface. The appropriate + datagram is issued. $\rightarrow$~\textit{FMMU} + +\item[FMMU] The FMMU configuration datagram was accepted. If the slave + has SDOs to configure, it is begun with sending the first SDO. + $\rightarrow$~\textit{SDO\_CONF} + + Otherwise, the slave can now be brought into the \textit{SAVEOP} + state. The state change state machine is started. + $\rightarrow$~\textit{SAVEOP} + +\item[SDO\_CONF] The CoE state machine is executed until termination. + $\rightarrow$~\textit{SDO\_CONF} + + If another SDO has to be configured, a new SDO download sequence is + begun. $\rightarrow$~\textit{SDO\_CONF} + + Otherwise, the slave can now be brought into the \textit{SAVEOP} + state. The state change state machine is started. + $\rightarrow$~\textit{SAVEOP} + +\item[SAVEOP] The state change state machine is executed until + termination. $\rightarrow$~\textit{SAVEOP} + + If the state change failed, the configuration has to be aborted. + $\rightarrow$~\textit{END} + + If the \textit{SAVEOP} state was the target state, the configuration + is finished. $\rightarrow$~\textit{END} + + The slave can now directly be brought into the \textit{OP} state and + the state change state machine is started a last time. + $\rightarrow$~\textit{OP} + +\item[OP] The state change state machine is executed until + termination. $\rightarrow$~\textit{OP} + + If the state change state machine terminates, the slave + configuration is finished, regardless of its success. + $\rightarrow$~\textit{END} + +\item[END] The termination state. + +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{The State Change State Machine} +\label{sec:fsm-change} +\index{FSM!State Change} + +The state change state machine, which can be seen in +figure~\ref{fig:fsm-change}, leads through the process of changing a +slave's state. This implements the states and transitions described in +\cite[section~6.4.1]{alspec}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.9\textwidth]{images/fsm-change} + \caption{Transition diagram of the state change state machine} + \label{fig:fsm-change} +\end{figure} + +\begin{description} +\item[START] The beginning state, where a datagram with the state + change command is written to the slave's ``AL Control Request'' + attribute. Nothing can fail. $\rightarrow$~\textit{CHECK} + +\item[CHECK] After the state change datagram has been sent, the ``AL + Control Response'' attribute is queried with a second datagram. + $\rightarrow$~\textit{STATUS} + +\item[STATUS] The read memory contents are evaluated: While the + parameter \textit{State} still contains the old slave state, the + slave is busy with reacting on the state change command. In this + case, the attribute has to be queried again. + $\rightarrow$~\textit{STATUS} + + In case of success, the \textit{State} parameter contains the new + state and the \textit{Change} bit is cleared. The slave is in the + requested state. $\rightarrow$~\textit{END} + + If the slave can not process the state change, the \textit{Change} + bit is set: Now the master tries to get the reason for this by + querying the \textit{AL Status Code} parameter. + $\rightarrow$~\textit{CODE} + +\item[END] If the state machine ends in this state, the slaves's state + change has been successful. + +\item[CODE] The status code query has been sent. Reading the + \textit{AL Status Code} might fail, because not all slaves support + this parameter. Anyway, the master has to acknowledge the state + change error by writing the current slave state to the ``AL Control + Request'' attribute with the \textit{Acknowledge} bit set. + $\rightarrow$~\textit{ACK} + +\item[ACK] After that, the ``AL Control Response'' attribute is + queried for the state of the acknowledgement. + $\rightarrow$~\textit{CHECK ACK} + +\item[CHECK ACK] If the acknowledgement has been accepted by the + slave, the old state is kept. Still, the state change was + unsuccessful. $\rightarrow$~\textit{ERROR} + + If the acknowledgement is ignored by the slave, a timeout happens. + In any case, the overall state change was unsuccessful. + $\rightarrow$~\textit{ERROR} + + If there is still now response from the slave, but the timer did not + run out yet, the slave's ``AL Control Response'' attribute is + queried again. $\rightarrow$~\textit{CHECK ACK} + +\item[ERROR] If the state machine ends in this state, the slave's + state change was unsuccessful. + +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{The SII State Machine} +\label{sec:fsm-sii} +\index{FSM!SII} + +The SII\index{SII} state machine (shown in figure~\ref{fig:fsm-sii}) +implements the process of reading or writing E$^2$PROM data via the +Slave Information Interface described in \cite[section~5.4]{alspec}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.9\textwidth]{images/fsm-sii} + \caption{Transition diagram of the SII state machine} + \label{fig:fsm-sii} +\end{figure} + +\begin{description} +\item[READ\_START] The beginning state for reading access, where the + read request and the requested address are written to the SII + attribute. Nothing can fail up to now. + $\rightarrow$~\textit{READ\_CHECK} + +\item[READ\_CHECK] When the SII read request has been sent + successfully, a timer is started. A check/fetch datagram is issued, + that reads out the SII attribute for state and data. + $\rightarrow$~\textit{READ\_FETCH} + +\item[READ\_FETCH] Upon reception of the check/fetch datagram, the + \textit{Read Operation} and \textit{Busy} parameters are checked: + \begin{itemize} + \item If the slave is still busy with fetching E$^2$PROM data into + the interface, the timer is checked. If it timed out, the reading + is aborted ($\rightarrow$~\textit{ERROR}), if not, the check/fetch + datagram is issued again. $\rightarrow$~\textit{READ\_FETCH} + + \item If the slave is ready with reading data, these are copied from + the datagram and the read cycle is completed. + $\rightarrow$~\textit{END} + \end{itemize} +\end{description} + +The write access states behave nearly the same: + +\begin{description} +\item[WRITE\_START] The beginning state for writing access, + respectively. A write request, the target address and the data word + are written to the SII attribute. Nothing can fail. + $\rightarrow$~\textit{WRITE\_CHECK} + +\item[WRITE\_CHECK] When the SII write request has been sent + successfully, the timer is started. A check datagram is issued, that + reads out the SII attribute for the state of the write operation. + $\rightarrow$~\textit{WRITE\_CHECK2} + +\item[WRITE\_CHECK2] Upon reception of the check datagram, the + \textit{Write Operation} and \textit{Busy} parameters are checked: + \begin{itemize} + \item If the slave is still busy with writing E$^2$PROM data, the + timer is checked. If it timed out, the operation is aborted + ($\rightarrow$~\textit{ERROR}), if not, the check datagram is + issued again. $\rightarrow$~\textit{WRITE\_CHECK2} + \item If the slave is ready with writing data, the write cycle is + completed. $\rightarrow$~\textit{END} + \end{itemize} +\end{description} + +%------------------------------------------------------------------------------ + +\section{Mailbox Protocol Implementations} +\index{Mailbox} + +The EtherCAT master implements the EoE and the CoE mailbox +protocols. See the below section for details. + +%------------------------------------------------------------------------------ + +\subsection{Ethernet-over-EtherCAT (EoE)} +\label{sec:eoeimp} +\index{EoE} + +The EtherCAT master implements the Ethernet-over-EtherCAT mailbox +protocol to enable the tunneling of Ethernet frames to special slaves, +that can either have physical Ethernet ports to forward the frames to, +or have an own IP stack to receive the frames. + +\paragraph{Virtual Network Interfaces} + +The master creates a virtual EoE network interface for every +EoE-capable slave. These interfaces are called \textit{eoeX}, where X +is a number provided by the kernel on interface registration. Frames +sent to these interfaces are forwarded to the associated slaves by the +master. Frames, that are received by the slaves, are fetched by the +master and forwarded to the virtual interfaces. + +This bears the following advantages: + +\begin{itemize} +\item Flexibility: The user can decide, how the EoE-capable slaves are + interconnected with the rest of the world. +\item Standard tools can be used to monitor the EoE activity and to + configure the EoE interfaces. +\item The Linux kernel's layer-2-bridging implementation (according to + the IEEE 802.1D MAC Bridging standard) can be used natively to + bridge Ethernet traffic between EoE-capable slaves. +\item The Linux kernel's network stack can be used to route packets + between EoE-capable slaves and to track security issues, just like + having physical network interfaces. +\end{itemize} + +\paragraph{EoE Handlers} + +The virtual EoE interfaces and the related functionality is encapsuled +in the \textit{ec\_eoe\_t} class (see section~\ref{sec:class-eoe}). +So the master does not create the network interfaces directly: This is +done inside the constructor of the \textit{ec\_eoe\_t} class. An +object of this class is called ``EoE handler'' below. An EoE handler +additionaly contains a frame queue. Each time, the kernel passes a new +socket buffer for sending via the interface's +\textit{hard\_start\_xmit()} callback, the socket buffer is queued for +transmittion by the EoE state machine (see below). If the queue gets +filled up, the passing of new socket buffers is suspended with a call +to \textit{netif\_stop\_queue()}. + +\paragraph{Static Handler Creation} + +The master creates a pool of EoE handlers at startup, that are coupled +to EoE-capable slaves on demand. The lifetime of the corresponding +network interfaces is equal to the lifetime of the master module. +This approach is opposed to creating the virtual network interfaces on +demand (i.~e. on running across a new EoE-capable slave). The latter +approach was considered as difficult, because of serveral reasons: + +\begin{itemize} +\item The \textit{alloc\_netdev()} function can sleep and must be + called from a non-interrupt context. This reduces the flexibility of + choosing an appropriate method for cyclic EoE processing. +\item Unregistering network interfaces requires them to be ``down'', + which can not be guaranteed upon sudden disappearing of an + EoE-capable slave. +\item The connection to the EoE-capable slaves must be as continuous + as possible. Especially the transition from idle to operation mode + (and vice versa) causes the rebuilding of the internal data + structures. These transitions must be as transparent as possible for + the instances using the network interfaces. +\end{itemize} + +\paragraph{Number of Handlers} + +The master module has a parameter \textit{ec\_eoeif\_count} to specify +the number of EoE interfaces (and handlers) per master to create. This +parameter can either be specified when manually loading the master +module, or (when using the init script) by setting the +\textit{\$EOE\_INTERFACES} variable in the sysconfig file (see +section~\ref{sec:sysconfig}). Upon loading of the master module, the +virtual interfaces become available: + +\begin{lstlisting} + host# `\textbf{ifconfig -a}` + eoe0 Link encap:Ethernet HWaddr 00:11:22:33:44:06 + BROADCAST MULTICAST MTU:1500 Metric:1 + RX packets:0 errors:0 dropped:0 overruns:0 frame:0 + TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1000 + RX bytes:0 (0.0 b) TX bytes:0 (0.0 b) + + eoe1 Link encap:Ethernet HWaddr 00:11:22:33:44:07 + BROADCAST MULTICAST MTU:1500 Metric:1 + RX packets:0 errors:0 dropped:0 overruns:0 frame:0 + TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 + collisions:0 txqueuelen:1000 + RX bytes:0 (0.0 b) TX bytes:0 (0.0 b) + ... +\end{lstlisting} + +\paragraph{Coupling of EoE Slaves} + +During execution of the slave scan state machine (see +section~\ref{sec:fsm-scan}), the master determines the supported +mailbox protocols. This is done by examining the ``Supported Mailbox +Protocols'' mask field at word address 0x001C of the SII\index{SII}. +If bit 1 is set, the slave supports the EoE protocol. After slave +scanning, the master runs through all slaves again and couples each +EoE-capable slave to a free EoE handler. It can happen, that there are +not enough EoE handlers to cover all EoE-capable slaves. In this case, +the number of EoE handlers must be increased accordingly. + +\paragraph{EoE State Machine} +\index{FSM!EoE} + +Every EoE handler owns an EoE state machine, that is used to send +frames to the coupled slave and receive frames from the it via the EoE +communication primitives. This state machine is showed in +figure~\ref{fig:fsm-eoe}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.7\textwidth]{images/fsm-eoe} + \caption{Transition diagram of the EoE state machine} + \label{fig:fsm-eoe} +\end{figure} + +\begin{description} +\item[RX\_START] The beginning state of the EoE state machine. A + mailbox check datagram is sent, to query the slave's mailbox for new + frames. $\rightarrow$~\textit{RX\_CHECK} + +\item[RX\_CHECK] The mailbox check datagram is received. If the + slave's mailbox did not contain data, a transmit cycle is started. + $\rightarrow$~\textit{TX\_START} + + If there are new data in the mailbox, a datagram is sent to fetch + the new data. $\rightarrow$~\textit{RX\_FETCH} + +\item[RX\_FETCH] The fetch datagram is received. If the mailbox data + do not contain a ``EoE Fragment request'' command, the data are + dropped and a transmit sequence is started. + $\rightarrow$~\textit{TX\_START} + + If the received Ethernet frame fragment is the first fragment, a new + socket buffer is allocated. In either case, the data are copied into + the correct position of the socket buffer. + + If the fragment is the last fragment, the socket buffer is forwarded + to the network stack and a transmit sequence is started. + $\rightarrow$~\textit{TX\_START} + + Otherwise, a new receive sequence is started to fetch the next + fragment. $\rightarrow$~\textit{RX\_\-START} + +\item[TX\_START] The beginning state of a transmit sequence. It is + checked, if the transmittion queue contains a frame to send. If not, + a receive sequence is started. $\rightarrow$~\textit{RX\_START} + + If there is a frame to send, it is dequeued. If the queue was + inactive before (because it was full), the queue is woken up with a + call to \textit{netif\_wake\_queue()}. The first fragment of the + frame is sent. $\rightarrow$~\textit{TX\_SENT} + +\item[TX\_SENT] It is checked, if the first fragment was sent + successfully. If the current frame consists of further fragments, + the next one is sent. $\rightarrow$~\textit{TX\_SENT} + + If the last fragment was sent, a new receive sequence is started. + $\rightarrow$~\textit{RX\_START} +\end{description} + +\paragraph{EoE Processing} + +To execute the EoE state machine of every active EoE handler, there +must be a cyclic process. The easiest thing would be to execute the +EoE state machines synchronously to the operation state machine (see +section~\ref{sec:fsm-op}) with every realtime cycle. This approach has +the following disadvantages: + +\begin{itemize} +\item Only one EoE fragment can be sent or received every few cycles. + This causes the data rate to be very low, because the EoE state + machines are not executed in the time between the realtime + cycles. Moreover, the data rate would be dependent on the frequency + of the realtime process. +\item The receiving and forwarding of frames to the kernel requires + the dynamic allocation of frames. Some realtime extensions do not + support calling memory allocation functions in realtime context, so + the EoE state machine may not be executed with each realtime cycle. +\end{itemize} + +To overcome these problems, an own cyclic process is needed to +asynchronously execute the EoE state machines. For that, the master +owns a kernel timer, that is executed each timer interrupt. This +guarantees a constant bandwidth, but poses the new problem of +concurrent access to the master. The locking mechanisms needed for +this are introduced in section~\ref{sec:concurr}. +Section~\ref{sec:concurrency} gives practical implementation examples. + +\paragraph{Idle Mode} + +EoE data must also be exchanged idle mode, to guarantee the continuous +availability of the connection to the EoE-capable slaves. Although +there is no realtime module connected in this case, the master is +still accessed by the idle state machine (see +section~\ref{sec:fsm-idle}), that is executed by the master's +workqueue. With the EoE timer running in addition, there is still +concurrency, that has to be protected by a lock. Therefore the master +owns an internal spinlock that is used protect master access during +idle mode. + +\paragraph{Automatic Configuration} + +By default, slaves are left in \textit{INIT} state during idle mode. +If an EoE interface is set to running state (i.~e. with the +\textit{ifconfig up} command), the requested slave state of the +related slave is automatically set to \textit{OP}, whereupon the idle +state machine will attempt to configure the slave and put it into +operation. + +%------------------------------------------------------------------------------ + +\subsection{CANopen-over-EtherCAT (CoE)} +\label{sec:coeimp} +\index{CoE} + +The CANopen-over-EtherCAT protocol \cite[section~5.6]{alspec} is used +to configure slaves on application level. Each CoE-capable slave +provides a list of SDOs for this reason. + +\paragraph{SDO Configuration} + +The SDO configurations have to be provided by the realtime module. +This is done via the \textit{ecrt\_slave\_conf\_sdo*()} methods (see +section~\ref{sec:ecrt-slave}), that are part of the realtime +interface. The slave stores the SDO configurations in a linked list, +but does not apply them at once. + +\paragraph{SDO Download State Machine} + +The best time to apply SDO configurations is during the slave's +\textit{PREOP} state, because mailbox communication is already +possible and slave's application will start with updating input data +in the succeeding \textit{SAVEOP} state. Therefore the SDO +configuration has to be part of the slave configuration state machine +(see section~\ref{sec:fsm-conf}): It is implemented via an SDO +download state machine, that is executed just before entering the +slave's \textit{SAVEOP} state. In this way, it is guaranteed that the +SDO configurations are applied each time, the slave is reconfigured. + +The transition diagram of the SDO Download state machine can be seen +in figure~\ref{fig:fsm-coedown}. + +\begin{figure}[htbp] + \centering + \includegraphics[width=.9\textwidth]{images/fsm-coedown} + \caption{Transition diagram of the CoE download state machine} + \label{fig:fsm-coedown} +\end{figure} + +\begin{description} +\item[START] The beginning state of the CoE download state + machine. The ``SDO Download Normal Request'' mailbox command is + sent. $\rightarrow$~\textit{REQUEST} + +\item[REQUEST] It is checked, if the CoE download request has been + received by the slave. After that, a mailbox check command is issued + and a timer is started. $\rightarrow$~\textit{CHECK} + +\item[CHECK] If no mailbox data is available, the timer is checked. + \begin{itemize} + \item If it timed out, the SDO download is aborted. + $\rightarrow$~\textit{ERROR} + \item Otherwise, the mailbox is queried again. + $\rightarrow$~\textit{CHECK} + \end{itemize} + + If the mailbox contains new data, the response is fetched. + $\rightarrow$~\textit{RESPONSE} + +\item[RESPONSE] If the mailbox response could not be fetched, the data + is invalid, the wrong protocol was received, or a ``Abort SDO + Transfer Request'' was received, the SDO download is aborted. + $\rightarrow$~\textit{ERROR} + + If a ``SDO Download Normal Response'' acknowledgement was received, + the SDO download was successful. $\rightarrow$~\textit{END} + +\item[END] The SDO download was successful. + +\item[ERROR] The SDO download was aborted due to an error. + +\end{description} + +%------------------------------------------------------------------------------ + +\section{User Space} +\label{sec:user} +\index{User space} + +For the master runs as a kernel module, accessing it is natively +limited to analyzing syslog messages and controlling using modutils. + +It is necessary to implement further interfaces, that make it easier +to access the master from user space and allow a finer influence. It +should be possible to view and to change special parameters at runtime. + +Bus visualization is a second point: For development and debugging +purposes it would be nice, if one could show the connected slaves with +a single command. + +Another aspect is automatic startup and configuration. If the master +is to be integrated into a running system, it must be able to +automatically start with a persistent configuration. + +A last thing is monitoring EtherCAT communication. For debugging +purposes, there had to be a way to analyze EtherCAT datagrams. The +best way would be with a popular network analyzer, like Wireshark +\cite{wireshark} (the former Ethereal) or others. + +This section covers all those points and introduces the interfaces and +tools to make all that possible. + +%------------------------------------------------------------------------------ + +\subsection{The Sysfs Interface} +\label{sec:sysfs} + +The system filesystem (Sysfs\index{Sysfs}) was introduced with Linux +kernel 2.5 and is a well-defined interface for drivers to export +information to user space. It serves also as an relief for the process +filesystem (Procfs), where over the years much non-process information +was concentrated. + +Sysfs exports information about devices, classes and busses via a +virtual filesystem, usually mounted to \textit{/sys}. The EtherCAT +master slightly differs from this concept, because the only physical +device is the network adapter it uses for bus communication, which is +already represented in Sysfs. For the EtherCAT bus is no system bus +like PCI (with device and driver structures), it would not make any +sense to represent it as bus structure in Sysfs. + +Therefore, the EtherCAT master is represented as a new directory +directly unter the Sysfs root. Every master gets its own Sysfs entry +named \textit{ethercatX}, where X is the index of the master. Two +masters would result in the directories \textit{/sys/ethercat0} and +\textit{/sys/ethercat1}, respectively. + +The Sysfs base class in the kernel code is the \textit{kobject} +structure. Each object structure, that is to be represented in Sysfs, +has to contain such a structure, because due to the concurrent access +(through ``normal'' kernel code and Sysfs code) the object deletion +gets a little more complicated: The object may not be freed until no +instance uses it any more. Therefore, each kobject maintains a +reference counter. If the reference counter gets zero, the object is +finally freed. A kobject can be registered to appear as a directory in +Sysfs with a call to \textit{kobject\_add()}. Each kobject type can +define attributes, that appear as files in the kobject's +directory. Callback functions have to be provided for reading (and +perhaps writing) access. + +\subsubsection{Master Attributes} +\label{sec:sysfs-master} + +Below is a typical listing of the masters Sysfs directory (that is a +file system representation of the master's kobject): + +\begin{lstlisting} + host> `\textbf{ls /sys/ethercat0}` + debug_level slave000 slave003 slave006 + eeprom_write_enable slave001 slave004 slave007 + info slave002 slave005 slave008 +\end{lstlisting} + +The following attributes exist in the master directory: + +\begin{description} +\item[debug\_level] (read/write) This attribute contains the master's + debug level, which controls, how much information is printed into + syslog. The values 0 (no additional debug messages), 1 (a few + additional debug messages) and 2 (all additional debug messages) are + defined. Writing is done with command like + + \begin{lstlisting}[gobble=4] + host# `\textbf{echo 1 > /sys/ethercat0/debug\_level}` + \end{lstlisting} + + and is receipted with a syslog message by the master: + + \begin{lstlisting}[gobble=4] + EtherCAT: Master debug level set to 1. + \end{lstlisting} + +\item[enable\_eeprom\_writing] (read/write) See + section~\ref{sec:eepromaccess} for how to use this attribute. + +\item[info] (read only) This attribute contains information about the + master. Example contents are below: + + \begin{lstlisting}[gobble=4] + host> `\textbf{cat /sys/ethercat0/info}` + + Mode: IDLE + Slaves: 9 + + Timing (min/avg/max) [us]: + Idle cycle: 4 / 4.38 / 34 + EoE cycle: 9 / 11.91 / 23 + + EoE statistics (RX/TX) [bps]: + eoe0: 0 / 3184 + \end{lstlisting} + + The mode can be \textit{ORPHANED}, \textit{IDLE} or + \textit{OPERATION}. The other parameters are self-explanatory. + +\end{description} + +\subsubsection{Domain Attributes} +\label{sec:sysfs-domain} + +In operation mode, each created domain is represented as a directory +\textit{domainX}, where X is the domain index. Below is a listing of +the domain directory contents: + +\begin{lstlisting} + host> `\textbf{ls /sys/ethercat0/domain0}` + image_size +\end{lstlisting} + +The domain directories currently only export the domain's image size. +It is planned to export the whole process data mapping for debugging +purposes. + +\subsubsection{Slave Attributes} +\label{sec:sysfs-slave} + +Each slave on the bus is represented in its own directory +\textit{slaveXXX}, where XXX is the slave's 3-digit ring position in +the EtherCAT bus. Below is a listing of a slave directory: + +\begin{lstlisting} + host> `\textbf{ls /sys/ethercat0/slave003}` + eeprom info state +\end{lstlisting} + +\begin{description} +\item[eeprom] (read/write) See section~\ref{sec:eepromaccess} for how + to use this attribute. + +\item[info] (read only) This attribute contains a bunch of information + about the slave. Below is an example output: + + \begin{lstlisting}[gobble=4] + host> `\textbf{cat /sys/ethercat0/slave003/info}` + + Name: EL4132 2K. Ana. Ausgang +/-10V + Vendor ID: 0x00000002 + Product code: 0x10243052 + + State: INIT + Ring position: 3 + Advanced position: 1:3 + + Data link status: + Port 0 (EBUS) Link down, Loop open, Signal detected + Port 1 (EBUS) Link down, Loop open, Signal detected + Port 2 (EBUS) Link down, Loop closed, No signal + Port 3 (EBUS) Link down, Loop closed, No signal + + Mailboxes: + RX mailbox: 0x1800/246, TX mailbox: 0x18F6/246 + Supported protocols: CoE, FoE + + SII data: + Group: AnaOut + Image: TERM_AO + Order#: EL4132 + + Sync-Managers: + 0: 0x1800, length 246, control 0x26, enable + 1: 0x18F6, length 246, control 0x22, enable + 2: 0x1000, length 0, control 0x24, enable + 3: 0x1100, length 0, control 0x20, enable + + PDOs: + RXPDO "Channel 1" (0x1600), Sync-Manager 2 + "Output" 0x6411:1, 16 bit + RXPDO "Channel 2" (0x1601), Sync-Manager 2 + "Output" 0x6411:2, 16 bit + \end{lstlisting} + + This is nearly all of the SII category information needed to + configure the slave, supplemented with state and addressing + information. + +\item[state] (read/write) This attribute contains the slave's state. + It can be read or written: + + \begin{lstlisting}[gobble=4] + host# `\textbf{cat /sys/ethercat0/slave003/state}` + OP + host# `\textbf{echo SAVEOP > /sys/ethercat0/slave003/state}` + \end{lstlisting} + + This command should also be receipted with a syslog message: + + \begin{lstlisting}[gobble=4] + EtherCAT: Accepted new state SAVEOP for slave 3. + EtherCAT: Changing state of slave 3 from OP to SAVEOP. + EtherCAT: Slave states: INIT, SAVEOP, OP. + \end{lstlisting} + + After the new requested state was accepted from user space, the + operation state machine (see section~\ref{sec:fsm-op}) or the idle + state machine (section~\ref{sec:fsm-idle}) notices, that the + requested slave state differs from the current one, and therefore + executes the slave configuration state machine, until the slave has + reached the requested state. +\end{description} + +%------------------------------------------------------------------------------ + +\subsubsection{E$^2$PROM Access} +\label{sec:eepromaccess} +\index{E$^2$PROM!Access} + +It is possible to directly read or write the complete E$^2$PROM +contents of the slaves. This was introduced for the reasons below: + +\begin{itemize} +\item The format of the E$^2$PROM data is still in development and + categories can be added in the future. With read and write access, + the complete memory contents can be easily backed up and restored. +\item Some E$^2$PROM data fields have to be altered (like the alias + address). A quick writing must be possible for that. +\item Through read access, analyzing category data is possible from + user space. +\end{itemize} + +Reading out E$^2$PROM data is as easy as reading other +attributes. Though the data are in binary format, analyzation is +easier with a tool like \textit{hexdump}: + +\begin{lstlisting} + host> `\textbf{cat /sys/ethercat0/slave003/eeprom | hexdump}` + 0000000 0103 0000 0000 0000 0000 0000 0000 008c + 0000010 0002 0000 3052 07f0 0000 0000 0000 0000 + 0000020 0000 0000 0000 0000 0000 0000 0000 0000 + ... +\end{lstlisting} + +Backing up E$^2$PROM contents gets as easy as copying a file: + +\begin{lstlisting} + host> `\textbf{cp /sys/ethercat0/slave003/eeprom slave003.eep}` +\end{lstlisting} + +Writing access is only possible as \textit{root}. Moreover writing has +to be explicitly enabled and is only allowed in idle mode. This is a +safety measure, because without the correct memory contents, a slave +is unusable. Writing E$^2$PROM contents in operation mode is not +provided yet. + +E$^2$PROM writing is enabled with the command below: + +\begin{lstlisting} + host# `\textbf{echo 1 > /sys/ethercat0/eeprom\_write\_enable}` +\end{lstlisting} + +The success can be seen in the syslog messages again: + +\begin{lstlisting} + EtherCAT: Slave EEPROM writing enabled. +\end{lstlisting} + +Now, it is possible to write E$^2$PROM contents to a slave. The master +will accept data through the \textit{eeprom} file and will perform a +short validation of the contents, before starting the write operation. +This validation checks the complete size and the category headers. + +\begin{lstlisting} + host# `\textbf{cat slave003.eep > /sys/ethercat0/slave003/eeprom}` +\end{lstlisting} + +The write operation can take a few seconds. + +\begin{lstlisting} + EtherCAT: EEPROM writing scheduled for slave 3, 88 words. + EtherCAT: Writing EEPROM of slave 3... + EtherCAT: Finished writing EEPROM of slave 3. +\end{lstlisting} + +%------------------------------------------------------------------------------ + +\subsection{User Space Tools} +\index{User space!Tools} + +There is a user space tool called \textit{lsec}\index{lsec} (``List +EtherCAT'') to visualize the EtherCAT bus. Running it usually results +in an output like this: + +\begin{lstlisting} + host> `\textbf{lsec}` + EtherCAT bus listing for master 0: + 0 1:0 OP EK1100 Ethernet Kopplerklemme (2A E-Bus) + 1 1:1 INIT EL4132 2K. Ana. Ausgang +/-10V + 2 1:2 INIT EL4132 2K. Ana. Ausgang +/-10V + 3 1:3 SAVEOP EL4132 2K. Ana. Ausgang +/-10V + 4 1:4 INIT EL5101 Incremental Encoder Interface + 5 1:5 INIT EL1014 4K. Dig. Eingang 24V, 10s + 6 1:6 OP EL6601 1 Port Switch (Ethernet, CoE) + 7 1:7 INIT EL5101 Incremental Encoder Interface + 8 1:8 INIT EL5001 1K. SSI Encoder +\end{lstlisting} + +Every slave is displayed as one text row. The first column shows its +ring position, the second displays the ``advanced position address'' +(see section~\ref{sec:addr}) and the third column displays the current +slave state. The last column is the slave's name, as it appears in the +``general'' E$^2$PROM category. + +The lsec program is a Perl script, that evaluates the Sysfs +\textit{info} attributes of the slaves (see +section~\ref{sec:sysfs-slave}). This is done for master $0$ by +default, but the master index can be specified via command line: + +\begin{lstlisting} + host> `\textbf{lsec -h}` + Usage: ec_list [OPTIONS] + -m Query master . + -h Show this help. +\end{lstlisting} + +This script has proved as useful for troubleshooting: If it displays +slaves, the master is up and running, and the bus connection is +present, too. It is also useful when building up a bus: It can verify +the list of slaves and help to create a process data image (see +chapter~\ref{chapter:usage}). + +%------------------------------------------------------------------------------ + +\subsection{System Integration} +\label{sec:system} + +To integrate the EtherCAT master into a running system, it has to be +guaranteed, that it is started on system startup. In addition, there has +to be a persistent configuration, that is also applied on startup. + +\subsubsection{The EtherCAT Init Script} +\label{sec:init} +\index{Init script} + +The EtherCAT master provides an ``init script'', that conforms to the +requirements of the ``Linux Standard Base'' (LSB\index{LSB}, +\cite{lsb}). The script is installed to \textit{/etc/init.d/ethercat}, +so that the master can be easily inserted as a service. The different +Linux distributions offer different ways to mark the service for +starting and stopping in certain runlevels (for example, SUSE provides +the \textit{insserv} command). + +To provide service dependencies (i.~e. which services have to be +started before) right inside the init script code, LSB defines a +special comment block. System tools can extract this information to +insert the EtherCAT init script at the correct place in the startup +sequence: + +\begin{lstlisting} + ### BEGIN INIT INFO + # Provides: ethercat + # Required-Start: $local_fs $syslog $network + # Should-Start: $time + # Required-Stop: $local_fs $syslog $network + # Should-Stop: $time + # Default-Start: 3 5 + # Default-Stop: 0 1 2 6 + # Short-Description: EtherCAT master modules + # Description: + ### END INIT INFO +\end{lstlisting} + +The init script can also be used for manually starting and stopping +the EtherCAT master. It has to be executed with one of the parameters +\textit{start}, \textit{stop}, \textit{restart} or \textit{status}. +Besides, a link to the script is placed at +\textit{/usr/sbin/rcethercat} for easier access. + +\begin{lstlisting} + host# `\textbf{/etc/init.d/ethercat restart}` + Shutting down EtherCAT master done + Starting EtherCAT master done +\end{lstlisting} + +\subsubsection{The EtherCAT Sysconfig File} +\label{sec:sysconfig} +\index{Sysconfig file} + +For persistent configuration, the init script uses a sysconfig file +installed to \textit{/etc/sys\-config/ethercat}, that is mandatory for +the init script. The sysconfig file contains all configuration +variables needed to operate a master: + +\begin{description} +\item[DEVICE\_INDEX] This variable must contain the PCI index of the + EtherCAT device. Setting this is mandatory for the EtherCAT init + script. Default: $-1$ +\item[EOE\_INTERFACES] The number of virtual Ethernet-over-EtherCAT + interfaces, every master creates on startup. See + section~\ref{sec:eoeimp}. Default: $0$ +\item[EOE\_BRIDGE] If this variable is set, all EoE interfaces will be + added to a network bridge according to IEEE 802.1D after master + startup. The variable must contain the name of the bridge. To use + this functionality, the kernel must be configured with the + \textit{CONFIG\_BRIDGE} option and the \textit{bridge-utils} package + must be installed (i.~e. the \textit{brctl} command is needed). +\item[EOE\_IP\_ADDRESS] The IP address of the EoE bridge. Setting this + together with \textit{EOE\_IP\_NETMASK} will let the local host + communicate with devices on the EoE bridge. +\item[EOE\_IP\_NETMASK] IP netmask of the EoE bridge. +\item[EOE\_EXTRA\_INTERFACES] The list of extra interfaces to include + in the EoE brid\-ge. Set this to interconnect the EoE bridge with + other local interfaces. If \textit{EOE\_\-BRIDGE} is empty or + undefined, setting this variable has no effect. Important: The IP + address of the listed interfaces will be cleared. Setting + \textit{EOE\_\-IP\_\-ADDRESS} and \textit{EOE\_IP\_NETMASK} will + re-enable them for IP traffic. +\item[EOE\_GATEWAY] The IP address of the default gateway. If this + variable is set, the gateway will be renewed after bridge + installation. This is necessary, if the default gateway's interface + is one of the \textit{EOE\_EXTRA\_INTERFACES}. +\end{description} + +%------------------------------------------------------------------------------ + +\subsection{Monitoring and Debugging} +\label{sec:debug} +\index{Monitoring} + +For debugging purposes, every EtherCAT master registeres a read-only +network interface \textit{ecX}, where X is a number, provided by the +kernel on device registration. While it is ``up'', the master forwards +every frame sent and received to this interface. + +This makes it possible to connect an network monitor (like Wireshark +or tcpdump) to the debug interface and monitor the EtherCAT frames. + +It has to be considered, that can be frame rate can be very high. The +idle state machine usually runs every kernel timer interrupt (up to +$1$~kHz) and with a connected realtime module, the rate can be even +higher. + +\paragraph{Attention:} The socket buffers needed for the operation of +the debugging interface have to be allocated dynamically. Some Linux +realtime extensions do not allow this in realtime context! + +%------------------------------------------------------------------------------ + +\section{Timing Aspects} +\label{sec:timing} + +Although EtherCAT's timing is highly deterministic and therefore +timing issues are rare, there are a few aspects that can (and should +be) dealt with. + +%------------------------------------------------------------------------------ + +\subsection{Realtime Interface Profiling} +\label{sec:timing-profile} +\index{Realtime!Profiling} + +One of the most important timing aspects are the runtimes of the +realtime interface functions, that are called in cyclic context. These +functions make up an important part of the overall timing of the +realtime module. To measure the timing of the functions, the following +code was used: + +\begin{lstlisting}[gobble=2,language=C] + c0 = get_cycles(); + ecrt_master_receive(master); + c1 = get_cycles(); + ecrt_domain_process(domain1); + c2 = get_cycles(); + ecrt_master_run(master); + c3 = get_cycles(); + ecrt_master_send(master); + c4 = get_cycles(); +\end{lstlisting} + +Between each call of an interface function, the CPU timestamp counter +is read. The counter differences are converted to microseconds with +help of the \textit{cpu\_khz} variable, that contains the number of +increments per millisecond. + +For the actual measuring, a system with a $2.0$~GHz CPU was used, that +ran the above code in an RTAI thread with a cycle time of $100$ +\textmu s. The measuring was repeated $n = 100$ times and the results +were averaged. These can be seen in table~\ref{tab:profile}. + +\begin{table}[htpb] + \centering + \caption{Profiling of a Realtime Cycle on a $2.0$~GHz Processor} + \label{tab:profile} + \vspace{2mm} + \begin{tabular}{l|r|r} + Element & Mean Duration [\textmu s] & Standard Deviancy [\textmu s] \\ + \hline + \textit{ecrt\_master\_receive()} & 8.04 & 0.48\\ + \textit{ecrt\_domain\_process()} & 0.14 & 0.03\\ + \textit{ecrt\_master\_run()} & 0.29 & 0.12\\ + \textit{ecrt\_master\_send()} & 2.18 & 0.17\\ \hline + Complete Cycle & 10.65 & 0.69\\ \hline + \end{tabular} +\end{table} + +It is obvious, that the the functions accessing hardware make up the +lion's share. The \textit{ec\_master\_receive()} executes the ISR of +the Ethernet device, analyzes datagrams and copies their contents into +the memory of the datagram objects. The \textit{ec\_master\_send()} +assembles a frame out of different datagrams and copies it to the +hardware buffers. Interestingly, this makes up only a quarter of the +receiving time. + +The functions that only operate on the masters internal data +structures are very fast ($\Delta t < 1$~\textmu s). Interestingly the +runtime of \textit{ec\_domain\_process()} has a small standard +deviancy relative to the mean value, while this ratio is about twice +as big for \textit{ec\_master\_run()}: This probably results from the +latter function having to execute code depending on the current state +and the different state functions are more or less complex. + +For a realtime cycle makes up about $10$~\textmu s, the theoretical +frequency can be up to $100$~kHz. For two reasons, this frequency +keeps being theoretical: + +\begin{enumerate} +\item The processor must still be able to run the operating system + between the realtime cycles. +\item The EtherCAT frame must be sent and received, before the next + realtime cycle begins. The determination of the bus cycle time is + difficult and covered in section~\ref{sec:timing-bus}. +\end{enumerate} + +%------------------------------------------------------------------------------ + +\subsection{Bus Cycle Measuring} +\label{sec:timing-bus} +\index{Bus cycle} + +For measuring the time, a frame is ``on the wire'', two timestamps +must be be taken: + +\begin{enumerate} +\item The time, the Ethernet hardware begins with physically sending + the frame. +\item The time, the frame is completely received by the Ethernet + hardware. +\end{enumerate} + +Both times are difficult to determine. The first reason is, that the +interrupts are disabled and the master is not notified, when a frame +is sent or received (polling would distort the results). The second +reason is, that even with interrupts enabled, the time from the event +to the notification is unknown. Therefore the only way to confidently +determine the bus cycle time is an electrical measuring. + +Anyway, the bus cycle time is an important factor when designing +realtime code, because it limits the maximum frequency for the cyclic +part of the realtime module. In practice, these timing parameters are +highly dependent on the hardware and often a trial and error method +must be used to determine the limits of the system. + +The central question is: What happens, if the cycle frequency is too +high? The answer is, that the EtherCAT frames that have been sent at +the end of the cycle are not yet received, when the next cycle starts. +First this is noticed by \textit{ecrt\_domain\_process()}, because the +working counter of the process data datagrams were not increased. The +function will notify the user via syslog\footnote{To limit syslog + output, a mechanism has been implementet, that outputs a summarized + notification at maximum once a second.}. In this case, the process +data keeps being the same as in the last cycle, because it is not +erased by the domain. When the domain datagrams are queued again, the +master notices, that they are already queued (and marked as sent). The +master will mark them as unsent again and output a warning, that +datagrams were ``skipped''. + +On the mentioned $2.0$~GHz system, the possible cycle frequency can be +up to $25$~kHz without skipped frames. This value can surely be +increased by choosing faster hardware. Especially the RealTek network +hardware could be replaced by a faster one. Besides, implementing a +dedicated ISR for EtherCAT devices would also contribute to increasing +the latency. These are two points on the author's to-do list. + +%------------------------------------------------------------------------------ + +\chapter{Using the EtherCAT Master} +\label{chapter:usage} +\index{Master!Usage} + +This chapter will give practical examples of how to use the EtherCAT +master via the realtime interface by writing a realtime module. + +Section~\ref{sec:make} shows how to compile and install the master, +while the sections~\ref{sec:mini} to~\ref{sec:concurrency} give +examples for different realtime modules. + +%------------------------------------------------------------------------------ + +\section{Compiling and Installing} +\label{sec:make} +\index{Master!Compilation} + +The current EtherCAT master code is available at~\cite{etherlab}. +After downloading the \textit{tar.bz2} file, it has to be unpacked +with the command below (or similar): + +\begin{lstlisting} + host> `\textbf{tar xjf ethercat-stable-1.1-r513-src.tar.bz2}` + host> `\textbf{cd ethercat-stable-1.1-r513-src}` +\end{lstlisting} + +For compilation, the kernel sources are needed. Basically any kernel +sources are appropriate\footnote{If a realtime extension is to be + used, the kernel has to be patched before that.}, that are +configured with networking. If the host kernel is not the running +kernel, a copy of the configuration template has to be made: + +\begin{lstlisting} + host> `\textbf{cp ethercat.conf.tmpl ethercat.conf}` +\end{lstlisting} + +Now the \textit{\$KERNEL} variable inside the \textit{ethercat.conf} +file can be adjusted to reflect the appropriate kernel version. If +everything is correct now, the sucessive call to + +\begin{lstlisting} + host> `\textbf{make}` +\end{lstlisting} + +will result in no errors. + +The following commands have to be entered as \textit{root}: To install +the kernel modules, the init script, the sysconfig file and the user +space tools, the below command has to be executed: + +\begin{lstlisting} + host# `\textbf{make install}` +\end{lstlisting} + +If the sysconfig file did not exist yet, the user is notified to edit +it, before running the master. For the contents of the file, see +section~\ref{sec:sysconfig}. To give a short summary: The most +important thing is to adjust the \textit{\$DEVICE\_INDEX} variable. It +has to be set to the index of the compatible network device to use +with EtherCAT, where the order of devices is dependent on their +position in the PCI bus. If this is not known, the index can be +determinded with trial and error, but it has to be considered that a +wrong value can cause a loss of network connection. + +After the basic configuration is done, the master can be started with +the below command: + +\begin{lstlisting} + host# `\textbf{/etc/init.d/ethercat start}` +\end{lstlisting} + +The operation of the master can be observed by looking at the +syslog\index{syslog} messages, which should look like the ones below: + +\begin{lstlisting}[numbers=left] + EtherCAT: Master driver, 1.1 (stable) - rev. 513, + compiled by fp at Aug 09 2006 10:23:20 + EtherCAT: Initializing 1 EtherCAT master(s)... + EtherCAT: Initializing master 0. + EtherCAT: Master driver initialized. + ec_8139too Fast Ethernet driver 0.9.27 Revision 513, + compiled by fp at Aug 09 2006 10:23:20 + ec_device_index is 0 + ACPI: PCI Interrupt 0000:01:00.0[A] -> Link [LNKC] + -> GSI 11 (level, low) -> IRQ 11 + ec0: RealTek RTL8139 at 0xd047c000, 00:c0:26:00:c6:aa, IRQ 11 + ec0: Identified 8139 chip type 'RTL-8100B/8139D' + Registering EtherCAT device... + Starting EtherCAT device... + EtherCAT: Link state changed to UP. + EtherCAT: Starting Idle mode. + EtherCAT: 11 slaves responding. + EtherCAT: Slave states: INIT, OP. + EtherCAT: Scanning bus. + EtherCAT: Bus scanning completed. + EtherCAT: No EoE handlers coupled. +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1}] The master module is loaded, + and one master is initialized. +\item[\normalfont\textcircled{\tiny 6}] The EtherCAT-capable RTL8139 + device driver is loaded. It connects its first network device to the + master. +\item[\normalfont\textcircled{\tiny 16}] The master starts idle mode + and begins scanning the bus for slaves. +\end{description} + +%------------------------------------------------------------------------------ + +\section{A Minimal Example Module} +\label{sec:mini} +\index{Examples!Minimal} + +This section will explain the usage of the EtherCAT master from a +minimal kernel module. The complete module code is obtainable as a +part of the EtherCAT master code release (see~\cite{etherlab}, file +\textit{examples/mini/mini.c}). + +The minimal example uses a kernel timer (software interrupt) to handle +cyclic code. After the timer function is executed, it re-adds itself +with a delay of one \textit{jiffy}\index{jiffies}, which results in a +timer frequency of \textit{HZ}\nomenclature{HZ}{Kernel macro + containing the timer interrupt frequency} + +The module-global variables, needed to operate the master can be seen +in listing~\ref{lst:minivar}. + +\begin{lstlisting}[language=C,numbers=left,caption={Minimal + variables},label=lst:minivar] + struct timer_list timer; + + ec_master_t *master = NULL; + ec_domain_t *domain1 = NULL; + + void *r_dig_in, *r_ana_out; + + ec_pdo_reg_t domain1_pdos[] = { + {"1", Beckhoff_EL1014_Inputs, &r_dig_in}, + {"2", Beckhoff_EL4132_Ouput1, &r_ana_out}, + {} + }; +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1}] There is a timer object + declared, that is needed to tell the kernel to install a timer and + execute a certain function, if it runs out. This is done by a + variable of the \textit{timer\_list} structure. +\item[\normalfont\textcircled{\tiny 3} -- \textcircled{\tiny 4}] There + is a pointer declared, that will later point to a requested EtherCAT + master. Additionally there is a pointer to a domain object needed, + that will manage process data IO. +\item[\normalfont\textcircled{\tiny 6}] The pointers \textit{r\_*} + will later point to the \underline{r}aw process data values inside + the domain memory. The addresses they point to will be set during a + call to \textit{ec\_\-master\_\-activate()}, that will create the + domain memory and configure the mapped process data image. +\item[\normalfont\textcircled{\tiny 8} -- \textcircled{\tiny 12}] The + configuration of the mapping of certain PDOs in a domain can easily + be done with the help of an initialization array of the + \textit{ec\_pdo\_reg\_t} type, defined as part of the realtime + interface. Each record must contain the ASCII bus-address of the + slave (see section~\ref{sec:addr}), the slave's vendor ID and + product code, and the index and subindex of the PDO to map (these + four fields can be specified in junction, by using one of the + defines out of the \textit{include/ecdb.h} header). The last field + has to be the address of the process data pointer, so it can later + be redirected appropriately. Attention: The initialization array + must end with an empty record (\textit{\{\}})! +\end{description} + +The initialization of the minimal realtime module is done by the +``Minimal init function'' in listing~\ref{lst:miniinit}. + +\begin{lstlisting}[language=C,numbers=left,caption={Minimal init + function},label={lst:miniinit}] + int __init init_mini_module(void) + { + if (!(master = ecrt_request_master(0))) { + goto out_return; + } + + if (!(domain1 = ecrt_master_create_domain(master))) { + goto out_release_master; + } + + if (ecrt_domain_register_pdo_list(domain1, + domain1_pdos)) { + goto out_release_master; + } + + if (ecrt_master_activate(master)) { + goto out_release_master; + } + + ecrt_master_prepare(master); + + init_timer(&timer); + timer.function = run; + timer.expires = jiffies + 10; + add_timer(&timer); + + return 0; + + out_release_master: + ecrt_release_master(master); + out_return: + return -1; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] It is tried to request the + first EtherCAT master (index 0). On success, the + \textit{ecrt\_\-request\_\-master()} function returns a pointer to + the reserved master, that can be used as an object to following + functions calls. On failure, the function returns \textit{NULL}. +\item[\normalfont\textcircled{\tiny 7}] In order to exchange process + data, a domain object has to be created. The + \textit{ecrt\_\-master\_\-create\_domain()} function also returns a + pointer to the created domain, or \textit{NULL} in error case. +\item[\normalfont\textcircled{\tiny 11}] The registration of domain + PDOs with an initialization array results in a single function call. + Alternatively the data fields could be registered with individual + calls of \textit{ecrt\_domain\_register\_pdo()}. +\item[\normalfont\textcircled{\tiny 16}] After the configuration of + process data mapping, the master can be activated for cyclic + operation. This will configure all slaves and bring them into + \textit{OP} state. +\item[\normalfont\textcircled{\tiny 20}] This call is needed to avoid + a case differentiation in cyclic operation: The first operation in + cyclic mode is a receive call. Due to the fact, that there is + nothing to receive during the first cycle, there had to be an + \textit{if}-statement to avoid a warning. A call to + \textit{ec\_master\_prepare()} sends a first datagram containing a + process data exchange datagram, so that the first receive call will + not fail. +\item[\normalfont\textcircled{\tiny 22} -- \textcircled{\tiny 25}] The + master is now ready for cyclic operation. The kernel timer that + cyclically executes the \textit{run()} function is initialized and + started. +\end{description} + +The coding of a cleanup function fo the minimal module can be seen in +listing~\ref{lst:miniclean}. + +\begin{lstlisting}[language=C,numbers=left,caption={Minimal cleanup + function},label={lst:miniclean}] + void __exit cleanup_mini_module(void) + { + del_timer_sync(&timer); + ecrt_master_deactivate(master); + ecrt_release_master(master); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] To cleanup the module, it it + necessary to stop the cyclic processing. This is done by a call to + \textit{del\_timer\_sync()} which safely removes a queued timer + object. It is assured, that no cyclic work will be done after this + call returns. +\item[\normalfont\textcircled{\tiny 4}] This call deactivates the + master, which results in all slaves being brought to their + \textit{INIT} state again. +\item[\normalfont\textcircled{\tiny 5}] This call releases the master, + removes any existing configuration and silently starts the idle + mode. The value of the master pointer is invalid after this call and + the module can be safely unloaded. +\end{description} + +The final part of the minimal module is that for the cyclic work. Its +coding can be seen in listing~\ref{lst:minirun}. + +\begin{lstlisting}[language=C,numbers=left,caption={Minimal cyclic + function},label={lst:minirun}] + void run(unsigned long data) + { + static uint8_t dig_in_0; + + ecrt_master_receive(master); + ecrt_domain_process(domain1); + + dig_in_0 = EC_READ_BIT(r_dig_in, 0); + EC_WRITE_S16(r_ana_out, dig_in_0 * 0x3FFF); + + ecrt_master_run(master); + ecrt_master_send(master); + + timer.expires += 1; // frequency = HZ + add_timer(&timer); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 5}] The cyclic processing starts + with receiving datagrams, that were sent in the last cycle. The + frames containing these datagrams have to be received by the network + interface card prior to this call. +\item[\normalfont\textcircled{\tiny 6}] The process data of domain 1 + has been automatically copied into domain memory while datagram + reception. This call checks the working counter for changes and + re-queues the domain's datagram for sending. +\item[\normalfont\textcircled{\tiny 8}] This is an example for reading + out a bit-oriented process data value (i.~e. bit 0) via the + \textit{EC\_READ\_BIT()} macro. See section~\ref{sec:macros} for + more information about those macros. +\item[\normalfont\textcircled{\tiny 9}] This line shows how to write a + signed, 16-bit process data value. In this case, the slave is able + to output voltages of $-10$~V to $+10$~V with a resolution of 16 + bit. This write command outputs either $0$~V or $+5$~V, depending + of the value of \textit{dig\_in\_0}. +\item[\normalfont\textcircled{\tiny 11}] This call runs the master's + operation state machine (see section~\ref{sec:fsm-op}). A single + state is processed, and datagrams are queued. Mainly bus observation + is done: The bus state is determined and in case of slaves that lost + their configuration, reconfiguration is tried. +\item[\normalfont\textcircled{\tiny 12}] This method sends all queued + datagrams, in this case the domain's datagram and one of the master + state machine. In best case, all datagrams fit into one frame. +\item[\normalfont\textcircled{\tiny 14} -- \textcircled{\tiny 15}] + Kernel timers are implemented as ``one-shot'' timers, so they have + to be re-added after each execution. The time of the next execution + is specified in \textit{jiffies} and will happen at the time of the + next system timer interrupt. This results in the \textit{run()} + function being executed with a frequency of \textit{HZ}. +\end{description} + +%------------------------------------------------------------------------------ + +\section{An RTAI Example Module} +\label{sec:rtai} +\index{Examples!RTAI} + +The whole code can be seen in the EtherCAT master code release +(see~\cite{etherlab}, file \textit{examples/rtai/rtai\_sample.c}). + +Listing~\ref{lst:rtaivar} shows the defines and global variables +needed for a minimal RTAI module with EtherCAT processing. + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI task + declaration},label={lst:rtaivar}] + #define FREQUENCY 10000 + #define TIMERTICKS (1000000000 / FREQUENCY) + + RT_TASK task; +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1} -- \textcircled{\tiny 2}] RTAI + takes the cycle period as nanoseconds, so the easiest way is to + define a frequency and convert it to a cycle time in nanoseconds. +\item[\normalfont\textcircled{\tiny 4}] The \textit{task} variable + later contains information about the running RTAI task. +\end{description} + +Listing~\ref{lst:rtaiinit} shows the module init function for the RTAI +module. Most lines are the same as in listing~\ref{lst:miniinit}, +differences come up when starting the cyclic code. + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI module init + function},label={lst:rtaiinit}] + int __init init_mod(void) + { + RTIME requested_ticks, tick_period, now; + + if (!(master = ecrt_request_master(0))) { + goto out_return; + } + + if (!(domain1 = ecrt_master_create_domain(master))) { + goto out_release_master; + } + + if (ecrt_domain_register_pdo_list(domain1, + domain1_pdos)) { + goto out_release_master; + } + + if (ecrt_master_activate(master)) { + goto out_release_master; + } + + ecrt_master_prepare(master); + + requested_ticks = nano2count(TIMERTICKS); + tick_period = start_rt_timer(requested_ticks); + + if (rt_task_init(&task, run, 0, 2000, 0, 1, NULL)) { + goto out_stop_timer; + } + + now = rt_get_time(); + if (rt_task_make_periodic(&task, now + tick_period, + tick_period)) { + goto out_stop_task; + } + + return 0; + + out_stop_task: + rt_task_delete(&task); + out_stop_timer: + stop_rt_timer(); + out_deactivate: + ecrt_master_deactivate(master); + out_release_master: + ecrt_release_master(master); + out_return: + return -1; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 24} -- \textcircled{\tiny 25}] The + nanoseconds are converted to RTAI timer ticks and an RTAI timer is + started. \textit{tick\_period} will be the ``real'' number of ticks + used for the timer period (which can be different to the requested + one). +\item[\normalfont\textcircled{\tiny 27}] The RTAI task is initialized + by specifying the cyclic function, the parameter to hand over, the + stack size, priority, a flag that tells, if the function will use + floating point operations and a signal handler. +\item[\normalfont\textcircled{\tiny 32}] The task is made periodic by + specifying a start time and a period. +\end{description} + +The cleanup function of the RTAI module in listing~\ref{lst:rtaiclean} +is nearly as simple as that of the minimal module. + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI module + cleanup function},label={lst:rtaiclean}] + void __exit cleanup_mod(void) + { + rt_task_delete(&task); + stop_rt_timer(); + ecrt_master_deactivate(master); + ecrt_release_master(master); + rt_sem_delete(&master_sem); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 2}] The RTAI task will be stopped + and deleted. +\item[\normalfont\textcircled{\tiny 3}] After that, the RTAI timer can + be stopped. +\end{description} + +The rest is the same as for the minimal module. + +Worth to mention is, that the cyclic function of the RTAI module +(listing~\ref{lst:rtairun}) has a slightly different architecture. The +function is not executed until returning for every cycle, but has an +infinite loop in it, that is placed in a waiting state for the rest of +each cycle. + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI module cyclic + function},label={lst:rtairun}] + void run(long data) + { + while (1) { + ecrt_master_receive(master); + ecrt_domain_process(domain1); + + k_pos = EC_READ_U32(r_ssi_input); + + ecrt_master_run(master); + ecrt_master_send(master); + + rt_task_wait_period(); + } + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] The \textit{while (1)} loop + executes for the lifetime of the RTAI task. +\item[\normalfont\textcircled{\tiny 12}] The + \textit{rt\_task\_wait\_period()} function sets the process into a + sleeping state until the beginning of the next cycle. It also + checks, if the cyclic function has to be terminated. +\end{description} + +%------------------------------------------------------------------------------ + +\section{Concurrency Example} +\label{sec:concurrency} +\index{Examples!Concurrency} + +As mentioned before, there can be concurrent access to the EtherCAT +master. The realtime module and a EoE\index{EoE} process can compete +for master access, for example. In this case, the module has to +provide the locking mechanism, because it depends on the module's +architecture which lock has to be used. The module makes this locking +mechanism available to the master through the master's locking +callbacks. + +In case of RTAI, the lock can be an RTAI semaphore, as shown in +listing~\ref{lst:convar}. A normal linux semaphore would not be +appropriate, because it could not block the RTAI task due to RTAI +running in a higher domain than the linux kernel (see~\cite{rtai}). + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI semaphore for + concurrent access},label={lst:convar}] + SEM master_sem; +\end{lstlisting} + +The module has to implement the two callbacks for requesting and +releasing the master lock. An exemplary coding can be seen in +listing~\ref{lst:conlock}. + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI locking + callbacks for concurrent access},label={lst:conlock}] + int request_lock(void *data) + { + rt_sem_wait(&master_sem); + return 0; + } + + void release_lock(void *data) + { + rt_sem_signal(&master_sem); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 1}] The \textit{request\_lock()} + function has a data parameter. The master always passes the value, + that was specified when registering the callback function. This can + be used for handing the master pointer. Notice, that it has an + integer return value (see line 4). +\item[\normalfont\textcircled{\tiny 3}] The call to + \textit{rt\_sem\_wait()} either returns at once, when the semaphore + was free, or blocks until the semaphore is freed again. In any case, + the semaphore finally is reserved for the process calling the + request function. +\item[\normalfont\textcircled{\tiny 4}] When the lock was requested + successfully, the function should return 0. The module can prohibit + requesting the lock by returning non-zero (see paragraph ``Tuning + the jitter'' below). +\item[\normalfont\textcircled{\tiny 7}] The \textit{release\_lock()} + function gets the same argument passed, but has a void return value, + because is always succeeds. +\item[\normalfont\textcircled{\tiny 9}] The \textit{rt\_sem\_signal()} + function frees the semaphore, that was prior reserved with + \textit{rt\_sem\_wait()}. +\end{description} + +In the module's init function, the semaphore must be initialized, and +the callbacks must be passed to the EtherCAT master: + +\begin{lstlisting}[language=C,numbers=left,caption={Module init + function for concurrent access},label={lst:coninit}] + int __init init_mod(void) + { + RTIME tick_period, requested_ticks, now; + + rt_sem_init(&master_sem, 1); + + if (!(master = ecrt_request_master(0))) { + goto out_return; + } + + ecrt_master_callbacks(master, request_lock, + release_lock, NULL); + // ... +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 5}] The call to + \textit{rt\_sem\_init()} initializes the semaphore and sets its + value to 1, meaning that only one process can reserve the semaphore + without blocking. +\item[\normalfont\textcircled{\tiny 11}] The callbacks are passed to + the master with a call to \textit{ecrt\_master\_callbacks()}. The + last parameter is the argument, that the master should pass with + each call to a callback function. Here it is not used and set to + \textit{NULL}. +\end{description} + +For the cyclic function being only one competitor for master access, +it has to request the lock like any other process. There is no need to +use the callbacks (which are meant for processes of lower priority), +so it can access the semaphore directly: + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI cyclic + function for concurrent access},label={lst:conrun}] + void run(long data) + { + while (1) { + rt_sem_wait(&master_sem); + + ecrt_master_receive(master); + ecrt_domain_process(domain1); + + k_pos = EC_READ_U32(r_ssi_input); + + ecrt_master_run(master); + ecrt_master_send(master); + + rt_sem_signal(&master_sem); + rt_task_wait_period(); + } + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 4}] Every access to the master has + to be preceeded by a call to \textit{rt\_sem\_wait()}, because + another instance might currently access the master. +\item[\normalfont\textcircled{\tiny 14}] When cyclic processing + finished, the semaphore has to be freed again, so that other + processes have the possibility to access the master. +\end{description} + +A little change has to be made to the cleanup function in case of +concurrent master access. + +\begin{lstlisting}[language=C,numbers=left,caption={RTAI module + cleanup function for concurrent access},label={lst:conclean}] + void __exit cleanup_mod(void) + { + rt_task_delete(&task); + stop_rt_timer(); + ecrt_master_deactivate(master); + ecrt_release_master(master); + rt_sem_delete(&master_sem); + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 7}] Upon module cleanup, the + semaphore has to be deleted, so that memory can be freed. +\end{description} + +\paragraph{Tuning the Jitter} +\index{Jitter} + +Concurrent access leads to higher jitter of the realtime process, +because there are situations, in which the realtime process has to +wait for a process of lower priority to finish accessing the master. +In most cases this is acceptable, because a master access cycle +(receive/process/send) only takes $10$~\textmu s to $20$~\textmu s on +recent systems, what would be the maximum additional jitter. However +some applications demand a minimum jitter. For this reason the master +access can be prohibited by the realtime module: If the time, another +process wants to access the master, is to close to the beginning of +the next realtime cycle, the module can disallow, that the lock is +taken. In this case, the request callback has to return $1$, meaning +that the lock has not been taken. The foreign process must abort its +master access and try again next time. + +This measure helps to significantly reducing the jitter produced by +concurrent master access. Below are exerpts of an example coding: + +\begin{lstlisting}[language=C,numbers=left,caption={Variables for + jitter reduction},label={lst:redvar}] + #define FREQUENCY 10000 // RTAI task frequency in Hz + // ... + cycles_t t_last_cycle = 0; + const cycles_t t_critical = cpu_khz * 1000 / FREQUENCY + - cpu_khz * 30 / 1000; +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 3}] The variable + \textit{t\_last\_cycle} holds the timer ticks at the beginning of + the last realtime cycle. +\item[\normalfont\textcircled{\tiny 4}] \textit{t\_critical} contains + the number of ticks, that may have passed since the beginning of the + last cycle, until there is no more foreign access possible. It is + calculated by substracting the ticks for $30$~\textmu s from the + ticks for a complete cycle. +\end{description} + +\begin{lstlisting}[language=C,numbers=left,caption={Cyclic function + with reduced jitter},label={lst:redrun}] + void run(long data) + { + while (1) { + t_last_cycle = get_cycles(); + rt_sem_wait(&master_sem); + // ... +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 4}] The ticks of the beginning of + the current realtime cycle are taken before reserving the semaphore. +\end{description} + +\begin{lstlisting}[language=C,numbers=left,caption={Request callback + for reduced jitter},label={lst:redreq}] + int request_lock(void *data) + { + // too close to the next RT cycle: deny access. + if (get_cycles() - t_last_cycle > t_critical) + return -1; + + // allow access + rt_sem_wait(&master_sem); + return 0; + } +\end{lstlisting} + +\begin{description} +\item[\normalfont\textcircled{\tiny 4}] If the time of request is too + close to the next realtime cycle (here: \textless~$30$~\textmu s + before the estimated beginning), the locking is denied. The + requesting process must abort its cycle. +\end{description} + +%------------------------------------------------------------------------------ + +\begin{thebibliography}{99} +\bibitem{etherlab} Ingenieurgemeinschaft IgH: EtherLab -- Open Source + Toolkit for rapid realtime code generation under Linux with + Simulink/RTW and EtherCAT technology. URL: http://etherlab.org, + July~31, 2006. +\bibitem{dlspec} IEC 61158-4-12: Data-link Protocol Specification. + International Electrotechnical Comission (IEC), 2005. +\bibitem{alspec} IEC 61158-6-12: Application Layer Protocol + Specification. International Electrotechnical Comission (IEC), 2005. +\bibitem{gpl} GNU General Public License, Version 2. URL: + http://www.gnu.org/licenses/gpl.txt. August~9, 2006. +\bibitem{lsb} Linux Standard Base. URL: + http://www.freestandards.org/en/LSB. August~9, 2006. +\bibitem{wireshark} Wireshark. URL: http://www.wireshark.org. + August~9, 2006. +\bibitem{automata} {\it Hopcroft, J.~E. / Ullman, J.~D.}: Introduction + to Automata Theory, Languages and Computation. Adison-Wesley, + Reading, Mass.~1979. +\bibitem{fsmmis} {\it Wagner, F. / Wolstenholme, P.}: State machine + misunderstandings. In: IEE journal ``Computing and Control + Engineering'', 2004. +\bibitem{rtai} RTAI. The RealTime Application Interface for Linux from + DIAPM. URL: http://www.rtai.org, 2006. +\end{thebibliography} + +\printglossary +\addcontentsline{toc}{chapter}{\nomname} +\markleft{\nomname} + +\printindex +\markleft{Index} + +%------------------------------------------------------------------------------ + +\end{document} + +%------------------------------------------------------------------------------