ted.dtx

%
% \iffalse  /!\  this file contains non-ascii --  encoding is utf-8 /!\
% 
%     ted package by Manuel P\'egouri\'e-Gonnard <mpg@elzevir.fr>
%     -----------------------------------------------------------
%
% This work may be distributed and/or modified under the conditions of
% the LaTeX Project Public License, either version 1.3c of this license
% or (at your option) any later version.  The latest version of this
% license is in:
%
%   http://www.latex-project.org/lppl.txt
%
% and version 1.3c or later is part of all distributions of LaTeX
% version 2006/05/20 or later.
%
% This work has the LPPL maintenance status `maintained'.
%
% The current maintainer of this work is Manuel P\'egouri\'e-Gonnard.
%
% This work consists of the file ted.dtx and the derived files ted.sty,
% ted.pdf and ted-fr.pdf.
%
%
%<*gobble>
\ProvidesFile{ted.dtx}
%</gobble>
%<package>\NeedsTeXFormat{LaTeX2e}
%<package>\ProvidesPackage{ted}
%<*package>
% \fi
% \ProvidesFile{ted.dtx}
  [2008/03/07 v1.06 \space a token list editor \space (mpg)]
% \iffalse
%</package>
% \fi
% \CheckSum{816}
% \CharacterTable
%  {Upper-case    \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z
%   Lower-case    \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z
%   Digits        \0\1\2\3\4\5\6\7\8\9
%   Exclamation   \!     Double quote  \"     Hash (number) \#
%   Dollar        \$     Percent       \%     Ampersand     \&
%   Acute accent  \'     Left paren    \(     Right paren   \)
%   Asterisk      \*     Plus          \+     Comma         \,
%   Minus         \-     Point         \.     Solidus       \/
%   Colon         \:     Semicolon     \;     Less than     \<
%   Equals        \=     Greater than  \>     Question mark \?
%   Commercial at \@     Left bracket  \[     Backslash     \\
%   Right bracket \]     Circumflex    \^     Underscore    \_
%   Grave accent  \`     Left brace    \{     Vertical bar  \|
%   Right brace   \}     Tilde         \~}
% \iffalse
%<*gobble>
\begingroup
%</gobble>
%<*batchfile>
\input docstrip
\keepsilent
\preamble

ted package by Manuel P\string\'egouri\string\'e-Gonnard %
<mpg@elzevir.fr>
This is a generated file. See ted.dtx for license information.

\endpreamble
\askforoverwritefalse
%\generate{\file{ted.ins}{\from{ted.dtx}{batchfile}}}
%\generate{\file{ted.drv}{\from{ted.dtx}{driver}}}
\generate{\file{ted.sty}{\from{ted.dtx}{package}}}
\generate{\file{ted-fr.drv}{\from{ted.dtx}{pilote}}}
%</batchfile>
%<*gobble>
\endgroup
%</gobble>
%<*driver|pilote>
\documentclass[oneside, a4paper]{ltxdoc}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage{fixltx2e, mparhack}
\usepackage[babel=true, expansion=false]{microtype}
\expandafter\newif\csname iffrenchdoc\endcsname % cachez ce if...
%<pilote>\frenchdoctrue
%</driver|pilote>
%<*gobble>
\iffrenchdoc
%</gobble>
%<*pilote>
\usepackage[lmargin=4.5cm, rmargin=3cm]{geometry}
\OnlyDescription
\usepackage[english, french]{babel}
\FrenchFootnotes \AddThinSpaceBeforeFootnotes
\newcommand*\eng[1]{\emph{\foreignlanguage{english}{#1}}}
\usepackage[bookmarks=false]{hyperref}
\hypersetup{%
  pdftitle=L'extension ted,
  pdfsubject=Un éditeur de liste de lexèmes pour LaTeX2e}
%</pilote>
%<*gobble>
\else
%</gobble>
%<*driver>
\AlsoImplementation
\usepackage[english]{babel}
\usepackage[bookmarks=true, bookmarksnumbered=true, bookmarksopen=true,
  bookmarksopenlevel=2]{hyperref}
\hypersetup{%
  pdftitle=The ted package,
  pdfsubject=A token list editor for LaTex2e}
\settowidth\MacroIndent{\rmfamily\scriptsize 000\ }
\setlength\MacroTopsep{0pt}
\setcounter{tocdepth}{1}
\usepackage[nohints, english]{minitoc}
\mtcsetpagenumbers{secttoc}{off}
\mtcsetrules{secttoc}{off}
\mtcsetdepth{secttoc}{3}
\setlength{\stcindent}{0pt}
\mtcsetfont{secttoc}{subsection}{\normalfont}
\newcommand\tocwithmini{%
  \dosecttoc[n]
  \setcounter{tocdepth}{1}\tableofcontents
  \setcounter{tocdepth}{3}\faketableofcontents}
%</driver>
%<*gobble>
\fi
%</gobble>
%<*driver|pilote>
\hypersetup{%
  colorlinks=true, linkcolor=black, urlcolor=black,
  pdfauthor=Manuel Pégourié-Gonnard}
\newcommand\pf{\textsf}
\newcommand\optstar{\meta{$*$}}
\newcommand\topcs[1]{\texorpdfstring{\cs{#1}}{#1}}
\begin{document}
  \DocInput{ted.dtx}
\end{document}
%</driver|pilote>
%<*package>
% \fi
%
% \GetFileInfo{ted.dtx}
%
% \title{\iffrenchdoc L'extension \pf{ted}\else The \pf{ted} package\fi}
% \author{Manuel Pégourié-Gonnard\\ 
%   \href{mailto:mpg@elzevir.fr}{mpg@elzevir.fr}}
% \date{\fileversion\ (\filedate)}
%
% \maketitle
% \iffrenchdoc\else \tocwithmini \fi
%
% \section{Introduction}
%
% \iffrenchdoc
%
% Le nom de l'extension \pf{ted} signifie \eng{token list editor}, soit
% en français \og éditeur de listes de lexèmes \fg{}. Comparé à
% \texttt{sed} dont son nom s'inspire, il n'est pas très puissant : il
% ne sait faire actuellement que deux choses avec les listes de lexèmes,
% à savoir les afficher en détail, et y opérer des substitutions.
% Toutefois, sa caractéristique principale est de travailler vraiment au
% niveau des lexèmes, et non des simples caractères.
%
% L'extension \pf{ted} est capable de faire des substitutions même à
% l'intérieur d'une paire d'accolades, et n'a pas de problèmes avec des
% lexèmes particuliers. En fait, elle est conçue pour fonctionner même
% avec des listes comportant des lexèmes très exotiques.
%
% \else
%
% Just like \texttt{sed} is a stream editor, \pf{ted} is a token list
% editor.  Actually, it is not as powerful as \texttt{sed}, but its
% main feature is that it really works with tokens, not only characters.
% At the moment, it can do only two things with token lists: display it
% with full information on each token, and perform substitutions (that
% is, replacing every occurrence of a sublist with another token list).
%
% The \pf{ted} package can perform substitutions inside groups, and
% don't forbid any token in the lists. Actually, \pf{ted} is designed to
% work well even if strange tokens (that is, unusual (charcode,
% \cs{catcode}) pairs or tokens with a confusing \cs{meaning}) occur in
% the list.
%
% \fi
%
% \section{Usage}
%
% \iffrenchdoc
%
% L'extension \pf{ted} définit essentiellement deux macros publiques :
% \cs{Substitute} et \cs{ShowTokens}. La première est la raison d'être
% de \pf{ted}, mais au cours du développement de l'extension, la partie
% la plus difficile et intéressante a été d'être capable d'écrire la
% deuxième. J'ai donc décidé de la proposer à l'utilisateur, car je
% pense qu'elle peut être utile, soit pour déboguer du code, soit pour
% apprendre \TeX.
%
% \subsection{\topcs{Substitute}}
%
% \DescribeMacro{\Substitute}
% La syntaxe de \cs{Substitute} est la suivante.
% \begin{quote}
% \cs{Substitute}\optstar\oarg{sortie}\marg{entrée}\marg{de}\marg{vers}
% \end{quote}
% Commençons par l'usage le plus simple : sans étoile ni argument
% optionel. Sous cette forme, \cs{Substitute} remplace chaque occurrence
% de la sous-liste \meta{de} par \meta{vers} dans l'\meta{entrée}, et
% place le résultat dans le registre à lexèmes \cs{ted@toks}. Comme
% \cs{Substitute} est sans doute destiné à être utilisé par des auteurs
% de classe ou d'extensions, le |@| dans le nom de ce registre ne
% devrait pas poser de problèmes la plupart du temps.
%
% Quoi qu'il en soit, vous pouvez changer facilement le nom du registre
% utilisé pour la sortie avec l'argument optionnel en spécifiant le nom
% d'un registre à lexèmes dans \meta{sortie}. En fait, vous voulez
% peut-être que la sortie soit placée dans une macro et non un registre
% : dans ce cas, vous pouvez spécifier par exemple \cs{def}\cs{macro}
% (ou \cs{long}\cs{def}\cs{macro} etc.) comme argument optionnel. Dans
% tous les cas, \meta{sortie}|{trucs}| doit être une syntaxe légale
% d'affectation de |truc| dans la sortie. Bien sûr, si vous demandez à
% ce que la sortie soit placée dans une macro, vous devez veiller à ce
% qu'elle ne contienne pas de dièses (caractère de paramètre) mal
% placés. Exemples :
%
% \begin{quote}
% |\Substitute{a#b#c}{a}{A}|\\
% |\newtoks\vostoks \Substitute[\vostoks]{a#b#c}{a}{A}|\\
% |\Substitute[\def\votremacro]{a#b#c}{#}{##}|
% \end{quote}
%
% La version étoilée de \cs{Substitute} a pour but de vous simplifier la
% vie quand vous ne voulez pas spécifier explicitement une liste de
% lexèmes en entrée, mais plutôt utiliser le contenu d'une macro ou d'un
% registre, en développant une fois son premier argument obligatoire
% avant de commencer à travailler dessus. Ceci vous évite d'avoir à
% placer vous-même des \cs{expandafter}, surtout dans le cas ou vous
% souhaitez utiliser l'argument optionnel. Cette fois, l'usage est
% inversé par rapport à l'argument optionnel : c'est la cas d'une macro
% qui est le plus naturel. Pour un registre de lexèmes, vous devez
% écrire le \cs{the} vous même : ainsi, votre \meta{entrée} pourra être
% de la forme \cs{macro}, ou \cs{the}\cs{toksreg}, ou toute autre chose
% dont le 1-développement donne ce que vous voulez. Par exemple :
%
% \begin{quote}
% |\def\abc{abccdef} \newtoks\abctoks \abctoks{abc}|\\
% |\Substitute{\abc}{cc}{C} %| donne |\abc|\\
% |\Substitute*{\abc}{cc}{C} %| donne |abCdef|\\
% |\Substitute*{\the\abctoks}{cc}{C} %| aussi
% \end{quote}
%
% La version avec deux étoiles a aussi pour but de vous épargner la
% peine de contrôler vous-même le développement des arguments. Elle
% fonctionne comme la version étoilée simple, mais développe cette fois
% ses trois arguments obligatoires une fois avant de commencer à
% travailler. La même remarque que pour la version étoilée simple
% s'applique concernant les macros et les registres à lexèmes. J'espère
% que ces trois cas (de zéro à deux étoiles) couvriront la plupart des
% usage courants. Pour une gestion plus souple du développement des
% arguments, attendez \LaTeX$3$ !
%
% Le comportement de \cs{Substitute} devrait être évident la plupart du
% temps. Seul un cas particulier mérite sans doute une petite remarque :
% dans le cas ou la liste \meta{de} est vide, la liste \meta{vers} est
% insérée entre chaque paire de lexèmes de l'\meta{entrée} : en
% particulier elle n'est pas insérée au début ni à la fin. Par exemple,
% |\Substitute{abc}{}{1}| placera |a1b1c| dans \cs{ted@toks}.
% 
% Enfin, il peut être utile de savoir qu'une fois que \cs{Substitute} a
% fini son travail, le nombre de substitutions effectuées est accessible
% dans le registre \cs{ted@count}. On peut ainsi s'en servir pour
% compter, par exemple, le nombre d'espaces (donc approximativement de
% mots) dans un texte, en faisant semblant d'opérer une substitution
% dessus.
%
% \subsection{\topcs{ShowTokens}}
%
% \DescribeMacro{\ShowTokens}
% La syntaxe de \cs{ShowTokens} est la suivante.
% \begin{quote}
% \cs{ShowTokens}\optstar\marg{liste}
% \end{quote}
% Dans sa forme simple, \cs{ShowTokens} se contente d'afficher la liste
% de lexèmes, un par ligne. Pour les lexèmes de type caractère,
% l'affichage comporte le caractère lui-même, suivi de l'indication de
% son code de catégorie sous la forme utilisée par \cs{show}.  Je
% rappelle ci-dessous (table~\ref{cc}) pour la commodité du lecteur les
% différents \cs{catcode}s possibles dans une liste de lexème, ainsi
% qu'un exemple et la façon dont ils sont affichés par \cs{ShowTokens}.
%
% \begin{table}
% \begin{tabular}{rcl}
% 1  & |{| & |(begin-group character {)|\\
% 2  & |}| & |(end-group character })|\\
% 3  & |$| & |(math shift character $)|\\
% 4  & |&| & |(alignment tab character &)|\\
% 6  & |#| & |(macro parameter character #)|\\
% 7  & |^| & |(superscript character ^)|\\
% 8  & |_| & |(subscript character _)|\\
% 10 & | | & |(blank space  )|\\
% 11 & |a| & |(the letter a)|\\
% 12 & |0| & |(the character 0)|\\
% 13 & |~| & |(active character=macro:->\nobreakspace {})|\\
% \end{tabular}
% \caption{Les \cs{catcode} : code, exemple, et description.}\label{cc}
% \end{table}
% 
% Pour les séquence de contrôles et les caractères actifs, le (début du)
% \cs{meaning} est également affiché, sans dépasser les~$80$ colonnes de
% large afin de ne pas perturber l'affichage par ligne. Il s'agit bien
% sûr du sens courant au moment de l'appel à \cs{ShowTokens}.
% 
% \DescribeMacro{\ShowTokensLogonly} \DescribeMacro{\ShowTokensOnline}
% Le comportement par défaut est d'afficher ces listes à la fois sur le
% terminal et dans le fichier log. Si vous préférez que l'affichage
% n'ait lieu que dans le fichier log, vous n'avez qu'à dire
% \cs{ShowTokensLogonly}. Si vous souhaitez ensuite revenir au
% comportement par défaut, dites seulement \cs{ShowTokensOnline}.
%
% La version étoilée de \cs{ShowTokens} fonctionne de la même façon que
% dans le cas de \cs{Substitute} en développant son argument une fois
% avant d'en commencer l'analyse. La même remarque que précédemment
% s'applique concernant les macros et les registres. Imaginons par
% exemple que vous vouliez vérifier, après la définition d'une macro,
% que vous avez bien les bons \cs{catcode}s :
%
% \begin{quote}
% |\begingroup \uccode`\~=32 \uppercase{\endgroup|\\
% |  \def\macro{1~2}}|\\
% |\ShowTokens*{\macro} %| donne à l'écran : [...]\\
% |1 (the character 1)|\\
% |  (active character=macro:-> )|\\
% |2 (the character 2)|
% \end{quote}
% 
% \medskip
% J'aimerais conclure par la remarque suivante : au cours de la
% conception de \pf{ted}, j'ai vraiment essayé de ne faire aucune
% hypothèse sur les lexèmes présents dans la liste. Ainsi, vous pouvez
% utiliser librement des accolades, des dièses, des espaces, des
% \cs{par}, des \cs{if}s, des \cs{bgroup} ou \cs{egroup} dans toutes les
% listes de lexèmes. À ma connaissance à ce jour, la seule limitation
% est que les séquences de contrôle qui apparaissent ne doivent pas être
% (ou être \cs{let}-égales à) des macros intimes de \pf{ted},
% c'est-à-dire celles commençant par \cs{ted@@}.
%
% \begin{quote}
% Vous savez maintenant tout ce qu'il y a à savoir sur l'utilisation de
% \pf{ted}. Si vous souhaitez vous pencher sur son implémentation, il
% vous faudra lire les commentaires en anglais car je n'ai pas eu le
% courage de documenter mon code en deux langues. 
% \end{quote}
%
% \begin{center}\large
% C'est tout pour cette fois ! \\
% Amusez-vous bien avec \LaTeX{} !
% \end{center}
%
% \else
%
% The \pf{ted} package provides two user macros: \cs{Substitute} and
% \cs{ShowTokens}. The first one is the primary goal of the package, but
% to be able to do the second was the more interesting part while
% writing the package. I made it into a user macro since I believe it
% can be useful for debugging things, or for learning \TeX.
%
% \subsection{\topcs{Substitute}}
%
% \DescribeMacro{\Substitute}
% The syntax of \cs{Substitute} is as follows.
% \begin{quote}
% \cs{Substitute}\optstar\oarg{output}\marg{input}\marg{from}\marg{to}
% \end{quote}
% Let's begin with the basics. Without star or optional argument, the
% \cs{Substitute} macro will replace each occurrence of the \meta{from}
% token list with \marg{to} in the \meta{input}, and put the result in
% the \cs{toks} register \cs{ted@toks}. This macro has a |@| in its
% name, but since I think the \cs{Substitute} macro will be essentially
% be used by class or package writers, this should be ok.
%
% Anyway, if you don't like this name, you can specify another one as
% \meta{output} using the optional argument. Your \meta{output} should
% be the name of a \cs{toks} register. If you want the output to be put
% in a macro, use \cs{def}\cs{macro} (or \cs{long}\cs{def}\cs{macro}
% or\ldots) as the optional argument. Anyway, \meta{output}\marg{stuff}
% must be a legal syntax for an assignment: using \cs{macro} as optional
% argument will not work (and may actually result in chaos). Of course,
% if you want your output to be placed in a macro, it should not contain
% improperly placed hash signs (that is, macro parameter tokens). 
%
% \begin{quote}
% |\Substitute{a#b#c}{a}{A}|\\
% |\newtoks\yourtoks \Substitute[\yourtoks]{a#b#c}{a}{A}|\\
% |\Substitute[\def\yourmacro]{a#b#c}{#}{##}|
% \end{quote}
%
% The one-starred form of \cs{Substitute} is meant to help you when your
% \meta{input} is not an explicit token list, but the contents of either
% a macro or a \cs{toks} register, by expanding once its first mandatory
% argument before proceeding. It spares you the pain of using
% \cs{expandafter}s, especially in case you want to use the optional
% argument too. This time, things are reversed compared to the optional
% argument : using a macro instead of a toks register is easier.
% Actually, with the starred form, the first argument can be \cs{macro}
% or \cs{the}\cs{toksreg}, or anything whose one-time expansion is the
% token list you want \cs{Substitute} to act upon.
%
% \begin{quote}
% |\def\abc{abccdef} \newtoks\abctoks \abctoks{abc}|\\
% |\Substitute{\abc}{cc}{C} %| gives |\abc|\\
% |\Substitute*{\abc}{cc}{C} %| gives |abCdef|\\
% |\Substitute*{\the\abctoks}{cc}{C} %| too
% \end{quote}
%
% The two-starred form is also meant to avoid you trouble with
% development. It expands its three mandatory arguments once before
% executing. The remark about macros and \cs{toks} register still
% holds. I hope this three cases (from zero to two stars) will suffice
% for most purpose. For a better handling of arguments expansion, wait
% for \LaTeX$3$!
%
% The action of \cs{Substitute} is pretty obvious most of the time.
% Maybe a particular case needs some precision: when \meta{from} is
% empty, then the \meta{to} list gets inserted between each two tokens
% of the \cs{input}, but not before the first one. For example,
% \cs{Substitute}|{abc}{}{1}| puts |a1b1c| in \cs{ted@toks}.
%
% Finally, it may be useful to know that, after \cs{Subsitute} finished
% its job, it leaves the number of replaced occurrences in the count
% register \cs{ted@count}. This can be used, for example, to count
% spaces (hence words) in a text, by making a fake substitution on it.
%
% \subsection{\topcs{ShowTokens}}
%
% \DescribeMacro{\ShowTokens}
% The syntax of \cs{ShowTokens} is as follows.
% \begin{quote}
% \cs{ShowTokens}\optstar\marg{list}
% \end{quote}
% In its simple form, \cs{ShowTokens} just shows the list, one token per
% line.  For characters tokens, its prints the character, and its
% category code in human-friendly form. For the sake of readability,
% here is (table~\ref{cc}) a reminder of the possible \cs{catcode}s,
% with an exemple and the way \cs{ShowTokens} displays them.
%
% \begin{table}
% \begin{tabular}{rcl}
% 1  & |{| & |(begin-group character {)|\\
% 2  & |}| & |(end-group character })|\\
% 3  & |$| & |(math shift character $)|\\
% 4  & |&| & |(alignment tab character &)|\\
% 6  & |#| & |(macro parameter character #)|\\
% 7  & |^| & |(superscript character ^)|\\
% 8  & |_| & |(subscript character _)|\\
% 10 & | | & |(blank space  )|\\
% 11 & |a| & |(the letter a)|\\
% 12 & |0| & |(the character 0)|\\
% 13 & |~| & |(active character=macro:->\nobreakspace {})|\\
% \end{tabular}
% \caption{Possible \cs{catcode}s: code, example, and description.}\label{cc}
% \end{table}
% 
% For control sequences and active characters, it
% also prints their current \cs{meaning} as a bonus, or only the
% beginning of it (ending with \cs{ETC.}) if it is more than one line
% ($80$ columns) long.
%
% \DescribeMacro{\ShowTokensLogonly}
% \DescribeMacro{\ShowTokensOnline}
% The default is to show this list both in the terminal and in the log
% file.  If you don't want it to be printed on the terminal, just say
% \cs{ShowTokensLogonly}. If you change your mind latter, you can
% restore the default behaviour with \cs{ShowTokensOnline}.
%
% The starred form of \cs{ShowTokens} works the same as for
% \cs{Substitute}: it expands its argument once before analysing and
% displaying it. The same remarks hold: use \cs{macro} or
% \cs{the}\cs{toksreg} in the argument.
%
% \begin{quote}
% |\begingroup \uccode`\~=32 \uppercase{\endgroup|\\
% |  \def\macro{1~2}}|\\
% |\ShowTokens*{\macro} %| prints on screen: [...]\\
% |1 (the character 1)|\\
% |  (active character=macro:-> )|\\
% |2 (the character 2)|
% \end{quote}
%
% \medskip
% I would like to conclude with the following remark: I have really
% tried to make sure \pf{ted}'s macros will work fine even with the
% wierdest token list.  In particular, you can freely use begin-group
% and end-group characters, hash signs, spaces, \cs{bgroup} and
% \cs{egroup}, \cs{par}, \cs{if}s, as well as exotic
% charcode-\cs{catcode} pairs in every argument of the macros. As far as
% I am aware, the only restriction is you should not use the very
% private macros of \pf{ted} (those beginning with \cs{ted@@}) in your
% token lists.
%
% \fi
%
% \StopEventually{}
%
% \section{Implementation}
%
% A important problem, when trying to substitute things in token lists,
% is to handle begin-group and end-group tokens, since they prevent us
% from to reading the tokens one by one, and tend to be difficult to
% handle individually.  Two more kinds of tokens are special: the space
% tokens, since they\footnote{Actually, only tokens with charcode $32$
% and \cs{catcode}~$10$ (i.e. $32_{10}$ tokens) are concerned.} cannot
% be grabbed as the non-delimited argument of a macro, and the parameter
% tokens (hash signs), since they cannot be part of the delimiters in
% the parameter text of a macro. From now on, ``special tokens'' thus
% denotes tokens with \cs{catcode} $1$, $2$, $6$ or $10$.
%
% To get rid of these problems, the \cs{Substitute} command proceeds in
% three steps. First, encode the input, replacing all special tokens
% with nice control sequences representing them, then do the actual
% substitution, and finally decode the output, replacing the special
% control sequences with the initial special tokens.
%
% Encoding is the hard part. The idea is to try reading the tokens one
% by one; for this we have two means: using a macro with one
% non-delimited argument, or something like \cs{let}. The former doesn't
% work well with \cs{catcode}~$1$, $2$ or $10$ tokens, and the later do
% not see the name of the token (its character code, or its name for a
% CS). So we need to use both \cs{futurelet}, a ``grabbing'' macro with
% argument, and \cs{string} in order to scan the tokens. Actually, the
% encoding proceeds in two passes: in the first, we try and detect the
% special tokens, storing their character codes for later use, then do
% the actual encoding in the last pass.
%
% Decoding also processes the tokens one by one, and is simpler, since
% special cases are already detected. There is, however, a trick with
% groups since, when we encounter a begin-group character, we have to
% wait for the corresponding end-group before adding the whole thing to
% the output. There is also a simpler version of decoding, for
% \cs{ShowTokens}, for screen/log output, with no need to use this
% trick, since it only outputs \cs{catcode}-$12$ characters. Finally,
% the substitution part uses a macro with delimited argument, defined on
% the fly.
%
% The code is divided as follows. 
% \par\vspace{-0.5\baselineskip}
% \secttoc
% \par\vspace{-0.5\baselineskip}
%
% \begin{macro}{\ted@toks}
% \begin{macro}{\ted@list}
% \begin{macro}{\ted@code}
% Before we begin, just allocate (or give a nice name to) a few
% registers.
%    \begin{macrocode}
\@ifdefinable\ted@toks{\newtoks\ted@toks}
\@ifdefinable\ted@list{\let\ted@list\toks@}
\@ifdefinable\ted@code{\let\ted@code\count@}
\@ifdefinable\ted@count{\newcount\ted@count}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Encoding}
%
% \begin{macro}{\ted@encloop}
% \begin{macro}{\ted@encloop@}
% The two passes use the same loop for reading the input almost token
% by token. This loop grabs the next token through a
% \cs{futurelet}\ldots
%    \begin{macrocode}
\newcommand\ted@encloop{%
  \futurelet\@let@token
  \ted@encloop@}
%    \end{macrocode}
% \ldots then looks at it with some \cs{ifx} and \cs{ifcat} (non nested,
% since the token could be an \cs{if} itself), in order to distinguish
% between three cases: normal token, end reached, or special token. In
% the later case, remember which kind of special token it is, using a
% numeric code.
%    \begin{macrocode}
\newcommand\ted@encloop@{%
  \let\next\ted@do@normal
  \ifx\@let@token\ted@@end
    \let\next\ted@gobble@end
  \fi
  \ifcat\noexpand\@let@token##%
    \ted@code0
    \let\next\ted@do@special
  \fi
  \ifcat\noexpand\@let@token\@sptoken
    \ted@code1
    \let\next\ted@do@special
  \fi
  \ifcat\noexpand\@let@token\bgroup
    \ted@code2
    \let\next\ted@do@special
  \fi
  \ifcat\noexpand\@let@token\egroup
    \ted@code3
    \let\next\ted@do@special
  \fi
  \next}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@@end}
% \begin{macro}{\ted@gobble@end}
% Here we used the following to detect the end, then gobble it when
% reached.
%    \begin{macrocode}
\newcommand\ted@@end{\ted@@end@}
%    \end{macrocode}
%    \begin{macrocode}
\@ifdefinable\ted@gobble@end{%
  \def\ted@gobble@end\ted@@end{}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\ted@sanitize}
% \begin{macro}{\ted@@active}
% Now, this detection method, with \cs{futurelet} and \cs{ifcat}, is
% unable to distinguish the following three cases for potential special
% tokens: (i) a ``true'' (explicit) special token, (ii) a CS
% \cs{let}-equal to a special token, (iii) an active character
% \cs{let}-equal to a special token.  While this is pre-scanning's job
% to detect the (ii) case, the (iii) can be easily got rid of by
% redefining locally all active characters.
%    \begin{macrocode}
\count@\catcode\z@ \catcode\z@\active
\newcommand\ted@sanitize{%
  \count@\z@ \@whilenum\count@<\@cclvi \do{%
    \uccode\z@\count@
    \uppercase{\let^^00\ted@@active}%
    \advance\count@\@ne}}
\catcode\z@\count@
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@@active{\ted@@active@}
%    \end{macrocode}
% This sanitizing macro also mark active characters by \cs{let}-ing them
% equal to \cs{ted@@active} in order to detect them easily later, for
% example while displaying on-screen token analysis.  All operations
% (scanning, replacing, display and decoding) are going to happen inside
% a group where \cs{ted@sanitize} has been executed, so that active
% characters are no longer an issue.  \end{macro} \end{macro}
%
% \begin{macro}{\ted@encode} \begin{macro}{\ted@do@normal}
% \begin{macro}{\ted@do@special} The \cs{ted@encode} macro is the master
% macro for encoding. It only initialise a few things and launches the
% two loops. We select one of the tree steps by \cs{let}-ing
% \cs{ted@do@normal} and \cs{ted@do@special} to the appropriate action.
%    \begin{macrocode}
\newcommand\ted@encode[1]{%
  \ted@list{}%
  \let\ted@do@normal\ted@gobble@encloop
  \let\ted@do@special\ted@scan@special
  \ted@encloop#1\ted@@end 
  \ted@toks{}%
  \let\ted@do@normal\ted@addtoks@encloop
  \let\ted@do@special\ted@special@out
  \ted@encloop#1\ted@@end
  \ted@assert@listempty}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@assert@listempty}
% After the last loop, \cs{ted@list} should be empty. If it's not, it
% means something very weird happened during the encoding procedure. I
% hope the code below will never be executed :)
%    \begin{macrocode}
\newcommand\ted@assert@listempty{%
  \edef\next{\the\ted@list}%
  \ifx\next\@empty \else
    \PackageError{ted}{%
      Assertion `\string\ted@list\space is empty' failed}{%
      This should not happen. Please report this bug to the author.
      \MessageBreak By the way, you're in trouble there... I'm sorry.}%
  \fi}
%    \end{macrocode}
% \end{macro}
%
% \subsubsection{Pre-scanning}
%
% \begin{macro}{\ted@gobble@encloop}
% For normal tokens, things are pretty easy: just gobble them!
%    \begin{macrocode}
\newcommand\ted@gobble@encloop{%
  \afterassignment\ted@encloop
  \let\@let@token= }
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@scan@special}
% For special tokens, it's harder. We must distinguish explicit
% character tokens from control sequences \cs{let}-equal to special
% tokens.  For this, we use \cs{string}, then grab the next character to
% see whether its code is \cs{escapechar} or not.  Actually, things are
% not this easy, for two reasons. First, we have to make sure the next
% character's code is not already \cs{escapechar} before the
% \cs{string}, by accident. For this purpose, we set \cs{escapechar}
% to~$0$ except if next character's code is also~$0$, in which case we
% prefer~$1$.
%    \begin{macrocode}
\count@\catcode\z@ \catcode\z@ 12
\newcommand\ted@scan@special{%
  \begingroup
  \escapechar\if\@let@token^^00 \@ne \else \z@ \fi
  \expandafter\ted@check@space\string}
\catcode\z@\count@
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@check@space}
% \begin{macro}{\ted@check@space@}
% Second, we have to handle carefully the case of the next token being
% the $32_{10}$ token, since we cannot grab this one with a macro. We
% are in this case if and only if the token we just \cs{string}ed was a
% character token with code $32$, and it is enough to check if next
% token's \cs{catcode} is $10$ in order to detect it, since it will be
% $12$ otherwise. In order to check this, we use \cs{futurelet} again
% for pre-scanning.
%    \begin{macrocode}
\newcommand\ted@check@space{%
  \futurelet\@let@token
  \ted@check@space@}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@check@space@{%
  \ifcat\@let@token\@sptoken
    \endgroup
    \ted@addlist{32}%
    \expandafter\ted@gobble@encloop
  \else
    \expandafter\ted@list@special
  \fi}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@list@special}
% Now that we got rid of this nasty space problem, we know for sure that
% the next token has \cs{catcode} $12$, so we can easily grab it as an
% argument, find its charcode, and decide whether the original token was
% a control sequence or not. Note the \cs{expandafter} over
% \cs{endgroup} trick, since we need to add the charcode to the list
% outside the group (opened for the modified \cs{escapechar}) though it
% was set inside. 
%    \begin{macrocode}
\newcommand*\ted@list@special[1]{%
  \ted@code`#1\relax
  \expandafter\expandafter\expandafter
  \endgroup
  \ifnum\ted@code=\escapechar
    \ted@addlist{\m@ne}%
  \else
    \expandafter\ted@addlist\expandafter{\the\ted@code}%
  \fi
  \ted@encloop}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@addlist}
% Here we used the following macro to add an element to the list, which
% is space-separated.
%    \begin{macrocode}
\newcommand*\ted@addlist[1]{%
  \ted@list\expandafter{\the\ted@list#1 }}
%    \end{macrocode}
% \end{macro}
%
% \subsubsection{Actually encoding}
%
% Remember that, before this last encoding pass, \cs{ted@encode} did the
% following:
% \begin{quote}
% \cs{let}\cs{ted@do@normal}\cs{ted@addtoks@encloop}\\
% \cs{let}\cs{ted@do@special}\cs{ted@special@out}
% \end{quote}
% \begin{macro}{\ted@addtoks@encloop}
% The first one is very easy : normal tokens are just grabbed as
% arguments and appended to the output, then the loop continues.
%    \begin{macrocode}
\newcommand\ted@addtoks@encloop[1]{%
  \ted@toks\expandafter{\the\ted@toks#1}%
  \ted@encloop}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@special@out}
% Special tokens need to be encoded, but before, just check if they are
% really special: they aren't if the corresponding code is $-1$.
%    \begin{macrocode}
\newcommand\ted@special@out{%
  \ifnum\ted@list@read=\m@ne
    \ted@list@advance
    \expandafter\ted@cs@clean
  \else
    \expandafter\ted@special@encode
  \fi}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@cs@clean}
% Even if the potentially special token was not a real one, we have work
% to do.  Indeed, in the first pass we did break it using a \cs{string},
% and thus we introduced some foreign tokens in the stream. Most of them
% are not important since they have \cs{catcode}~$12$. Anyway, some of
% them may be space tokens : in this case we have extra $32$'s in our
% list. So, we need to check this before going any further.
%    \begin{macrocode}
\newcommand\ted@cs@clean[1]{%
  \expandafter\ted@add@toks{#1}%
  \expandafter\ted@cscl@loop\string#1 \@nil}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@cscl@loop}
% We first add the CS to the output, then break it with a \cs{string} in
% order to look at its name with the following loop. It first grabs
% everything to the first space\ldots
%    \begin{macrocode}
\@ifdefinable\ted@cscl@loop{%
  \def\ted@cscl@loop#1 {%
    \futurelet\@let@token
    \ted@cscl@loop@}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@cscl@loop@}
% \ldots and carefully look at the next token in order to know if we are
% finished or not. 
%    \begin{macrocode}
\newcommand\ted@cscl@loop@{%
  \ifx\@let@token\@nil 
    \expandafter\ted@gobble@encloop
  \else
    \ted@list@advance
    \expandafter\ted@cscl@loop
  \fi}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\ted@special@encode}
% Now, let's come back to the special tokens. As we don't need the token
% to encode it (we already know its \cs{catcode} from \cs{ted@code},
% and its charcode is stored in the list), we first gobble it in order
% to prepare for next iteration.
%    \begin{macrocode}
\newcommand\ted@special@encode{%
    \afterassignment\ted@special@encode@
  \let\@let@token= }
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@special@encode@}
% Then we encode it in two steps : first, create a control sequence with
% name \cs{ted@@}\meta{code}\meta{charcode}, where code is a digit
% denoting\footnote{I don't store the \cs{catcode} for two reasons:
% first, having a single digit is easier; second, having the true
% catcode would be useless (though it could maybe make the code more
% readable).} the \cs{catcode} of the special token, \ldots
%    \begin{macrocode}
\newcommand\ted@special@encode@{%
  \expandafter\ted@special@encode@@\expandafter{%
    \csname ted@@\the\ted@code\ted@list@read\endcsname}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@special@encode@@}
% \begin{macro}{\ted@@special}
% \ldots then, mark this CS as a special token encoding, in order to
% make it easier to detect later, add it to the output and loop again.
%    \begin{macrocode}
\newcommand*\ted@special@encode@@[1]{%
  \ted@list@advance
  \let#1\ted@@special
  \ted@addtoks@encloop{#1}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@@special{\ted@@special@}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% 
% \begin{macro}{\ted@list@read}
% \begin{macro}{\ted@list@read@}
% Here we used the following macros in order to manage our charcode list.
% The reading one is fully expandable.
%    \begin{macrocode}
\newcommand\ted@list@read{%
  \expandafter\ted@list@read@\the\ted@list\@nil}
%    \end{macrocode}
%    \begin{macrocode}
\@ifdefinable\ted@list@read@{%
  \def\ted@list@read@#1 #2\@nil{%
    #1}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@list@advance}
% \begin{macro}{\ted@list@advance@}
% Since it's expandable, it cannot change the list, so we need a
% separate macro to remove the first element from the list, once read.
%    \begin{macrocode}
\newcommand\ted@list@advance{%
  \expandafter\ted@list@advance@\the\ted@list\@nil}
%    \end{macrocode}
%    \begin{macrocode}
\@ifdefinable\ted@list@advance@{
  \def\ted@list@advance@#1 #2\@nil{%
    \ted@list{#2}}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \subsection{Decoding}
%
% \begin{macro}{\ted@add@toks}
% Main decoding macro is \cs{ted@decode}. It is again a loop, processing
% the token list one by one. For normal tokens, things are easy as
% always: just add them to the output, via
%    \begin{macrocode}
\newcommand\ted@add@toks[1]{%
  \ted@toks\expandafter{\the\ted@toks#1}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@decode}
% Encoded special tokens are easily recognized, since they were \cs{let}
% equal to \cs{ted@@special}. In order to decode it, we use the name of
% the CS. The following macro uses \LaTeX-style \cs{if} in order to
% avoid potential nesting problems when \cs{ifs} are present in the
% token list being processed.
%    \begin{macrocode}
\newcommand\ted@decode[1]{%
  \ifx#1\ted@@end \expandafter\@gobble\else\expandafter\@firstofone\fi{%
    \ifx#1\ted@@special
      \expandafter\@firstoftwo
    \else
      \expandafter\@secondoftwo
    \fi{%
      \begingroup \escapechar\m@ne \expandafter\endgroup
      \expandafter\ted@decode@special\string#1\@nil
      }{%
      \ted@add@toks{#1}}%
    \ted@decode}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@decode@special}
% The next macro should then gobble the |ted@@| part of the CS name, and
% use the last part as two numeric codes (here we use the fact that the
% first one is only a digit). 
%    \begin{macrocode}
\@ifdefinable\ted@decode@special{%
  \begingroup\escapechar\m@ne \expandafter\endgroup\expandafter
  \def\expandafter\ted@decode@special\string\ted@@#1#2\@nil{%
%    \end{macrocode}
% It then proceeds according to the first code, building back the
% original token and adding it to the output. The first two kinds of
% tokens (macro parameter characters and blank spaces) are easily dealt
% with.
%    \begin{macrocode}
    \ifcase#1
      \begingroup \uccode`##=#2 \uppercase{\endgroup
        \ted@add@toks{##}}%
    \or
      \begingroup \uccode32=#2 \uppercase{\endgroup
        \ted@add@toks{ }}%
    \or
%    \end{macrocode}
% For begin-group and end-group characters, we have a problem, since
% they are impossible to handle individually: we can only add a
% \meta{balanced text} to the output. So, when we find a begin-group
% character, we just open a group (a real one), and start decoding again
% inside the group, until we find the corresponding end-group character.
% Then, we enclose the local decoded list of tokens into the correct
% begin-group/end-group pair, and then add it to the output one group
% level below, using the \cs{expandafter}-over-\cs{endgroup} trick
% (essential here).
%    \begin{macrocode}
      \begingroup \ted@toks{}%
      \uccode`{=#2
    \or
      \uccode`}=#2
      \uppercase{\ted@toks\expandafter{\expandafter{\the\ted@toks}}}%
      \expandafter\endgroup
      \expandafter\ted@add@toks\expandafter{\the\ted@toks}%
    \fi}}
%    \end{macrocode}
% \end{macro}
%
% \subsection{Substitution}
%
% For this part, the idea\footnote{for which I am grateful to Jean-Côme
% Charpentier, who first taught me the clever use delimited arguments
% (and lots of other wonderful things) in \nolinkurl{fr.comp.text.tex}}
% is to use a macro whose first argument is delimited with the
% \meta{from} string, which outputs the first argument followed by the
% \meta{to} string, and loops. Obviously this macro has to be defined on
% the fly. All tokens lists need to be encoded first, and the output
% decoded at end. Since all this needs to happens inside a group (for
% \cs{ted@sanitize} and the marking up of special-characters control
% sequences), remember to ``export'' \cs{ted@toks} when done.
%
% \begin{macro}{\ted@Substitude}
% The main substitution macro is as follows. Arguments are \meta{input},
% \meta{from}, \meta{to}.  \cs{ted@output} will be discussed later.
%    \begin{macrocode}
\newcommand\ted@Substitute[3]{%
  \begingroup \ted@sanitize
  \ted@encode{#3}%
  \expandafter\ted@def@subsmac\expandafter{\the\ted@toks}{#2}%
  \ted@encode{#1}%
  \ted@subsmac
  \ted@toks\expandafter{\expandafter}%
  \expandafter\ted@decode\the\ted@toks\ted@@end
  \expandafter\endgroup
  \expandafter\ted@output\expandafter{\the\ted@toks}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@def@subsmac}
% The actual iterative substitution macro is defined by the following
% macro, whose arguments are the \meta{to} string, encoded, and the
% plain \meta{from} string.
%    \begin{macrocode}
\newcommand\ted@def@subsmac[2]{%
  \ted@encode{#2}%
  \long\expandafter\def\expandafter\ted@subsmac@loop
  \expandafter##\expandafter1\the\ted@toks##2{%
    \ted@add@toks{##1}%
    \ifx##2\ted@@end 
      \expandafter\@firstoftwo
    \else
      \expandafter\@secondoftwo
    \fi{%
      \expandafter\ted@remove@nil\the\ted@toks
      }{%
      \global\advance\ted@count\@ne
      \ted@add@toks{#1}\ted@subsmac@loop##2}}%
  \expandafter\ted@def@subsmac@\expandafter{\the\ted@toks}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@def@subsmac@}
% While we have the encoded \meta{from} string at hand, define the
% start-loop macro.
%    \begin{macrocode}
\newcommand\ted@def@subsmac@[1]{%
  \def\ted@subsmac{%
    \global\ted@count\z@
    \ted@toks\expandafter{\expandafter}%
    \expandafter\ted@subsmac@loop\the\ted@toks\ted@@nil#1\ted@@end}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@remove@nil}
% You probably noticed the \cs{ted@@nil} after \cs{ted@toks} in the
% above definition. This is to avoid problems while trying to
% substitute something like ``AA'' in a list ending with ``A'' (new in
% v1.05). We need to remove it when finished.
%    \begin{macrocode}
\@ifdefinable\ted@remove@nil{%
  \long\def\ted@remove@nil#1\ted@@nil{%
    \ted@toks{#1}}}
%    \end{macrocode}
% \end{macro}
%
% \subsection{Display}
% 
% \begin{macro}{\ted@ShowTokens}
% In order to display the tokens one by one, we first encode the string.
%    \begin{macrocode}
\newcommand\ted@ShowTokens[1]{%
  \begingroup \ted@sanitize
  \ted@toks{#1}%
  \ted@typeout{--- Begin token decomposition of:}%
  \ted@typeout{\@spaces \the\ted@toks}%
  \ted@encode{#1}%
  \expandafter\ted@show@toks\the\ted@toks\ted@@end
  \endgroup
  \ted@typeout{--- End token decomposition.}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@show@toks}
% Then we proceed, almost like decoding, iteratively, processing the
% encoded tokens one by one. We detect control sequences the same way as
% in pre-scanning. For our tests (and also for use in
% \cs{ted@show@toks@}) we embed |#1| into \cs{ted@toks} in order to
% nest the \cs{if}s without fear. There are four cases that need to be
% typeset in different ways : active character, CS that represent a
% special token, normal CS, normal character token. However, we need to
% do one more test to detect the character tokens whose charcode is
% $32$, before we apply \cs{string} to it in order to check if it was a
% control sequence.
%    \begin{macrocode}
\count@\catcode\z@ \catcode\z@ 12
\newcommand\ted@show@toks[1]{%
  \ted@toks{#1}\expandafter
  \ifx\the\ted@toks\ted@@end \else\expandafter
    \ifx\the\ted@toks\ted@@active
%    \end{macrocode}
% It's time to think about the following: we are inside a group where
% all active characters were redefined, but we nonetheless want to
% display their meaning. In order to do this, the display need to
% actually happen after the current group is finished. For this we use
% \cs{aftergroup} (with specialized macro for displaying each kind of
% token).
%    \begin{macrocode}
      \aftergroup\ted@type@active
      \expandafter\aftergroup\the\ted@toks
    \else
      \if\expandafter\noexpand\the\ted@toks\@sptoken
        \aftergroup\ted@type@normal
        \expandafter\aftergroup\the\ted@toks
      \else
        \begingroup
        \escapechar\if\noexpand#1^^00 \@ne \else \z@ \fi
        \expandafter\expandafter\expandafter\ted@show@toks@
        \expandafter\string\the\ted@toks\@nil
      \fi
    \fi
    \expandafter\ted@show@toks
  \fi}
\catcode\z@\count@
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@show@toks@}
% Now test the remaining cases : special CS, normal CS, or normal
% character. 
%    \begin{macrocode}
\@ifdefinable\ted@show@toks@{%
  \long\def\ted@show@toks@#1#2\@nil{%
    \expandafter\endgroup
    \ifnum`#1=\escapechar
      \expandafter\ifx\the\ted@toks\ted@@special
        \ted@show@special#2\@nil
      \else
        \aftergroup\ted@type@cs
        \expandafter\aftergroup\the\ted@toks
      \fi
    \else
      \aftergroup\ted@type@normal
      \expandafter\aftergroup\the\ted@toks
    \fi}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@show@special}
% Let's begin our tour of specialized display macro with the most
% important one: \cs{ted@show@special}. Displaying the special token
% goes mostly the same way as decoding them, but is far easier, since we
% don't need to care about groups: display is done with
% \cs{catcode}~$12$ characters.
%    \begin{macrocode}
\@ifdefinable\ted@show@special{%
  \begingroup\escapechar\m@ne \expandafter\endgroup
  \expandafter\def\expandafter\ted@show@special\string\ted@@#1#2\@nil{%
    \ifcase#1
      \aftergroup\ted@type@hash
    \or
      \aftergroup\ted@type@blank
    \or
      \aftergroup\ted@type@bgroup
    \or
      \aftergroup\ted@type@egroup
    \fi
    \begingroup \uccode`1#2
    \uppercase{\endgroup\aftergroup1}}}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@type@hash}
% \begin{macro}{\ted@type@blank}
% \begin{macro}{\ted@type@bgroup}
% \begin{macro}{\ted@type@egroup}
% \begin{macro}{\ted@type@normal}
% The four macros for special tokens are obvious. So is the macro for
% normal tokens. By the way, \cs{ted@typeout} will be discussed in the
% next section.
%    \begin{macrocode}
\newcommand\ted@type@hash[1]{%
  \ted@typeout{#1 (macro parameter character #1)}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@type@blank[1]{%
  \ted@typeout{#1 (blank space #1)}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@type@bgroup[1]{%
  \ted@typeout{#1 (begin-group character #1)}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@type@egroup[1]{%
  \ted@typeout{#1 (end-group character #1)}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@type@normal[1]{%
  \ted@typeout{#1 (\meaning#1)}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@type@cs}
% \begin{macro}{\ted@type@active}
% For control sequences and active characters, we use more sophisticated
% macros. Indeed, their \cs{meaning} can be quite long, and since it is
% not so important (\pf{ted}'s work is lexical analysis, displaying the
% \cs{meaning} is just an add-on), we cut it so that lines are shorter
% than~$80$ colons, in order to save our one-token-a-line presentation.
%    \begin{macrocode}
\newcommand\ted@type@cs[1]{%
  \ted@type@long{\string#1 (control sequence=\meaning#1}}%
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@type@active[1]{%
  \ted@type@long{\string#1 (active character=\meaning#1}}%
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@type@long}
% Lines are cut and displayed by \cs{ted@type@long}. This macro uses a
% loop, counting down how many columns remain on the current line. The
% input need to be fully expanded first, and the output is stored in
% \cs{ted@toks}.
%    \begin{macrocode}
\newcommand\ted@type@long[1]{%
  \ted@toks{}%
  \ted@code72 
  \edef\next{#1}%
  \expandafter\ted@tl@loop\next\@nil}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@tl@loop}
% The only difficult thing in this loop is to take care of space tokens.
% For this we use again our \cs{futurelet} trick:
%    \begin{macrocode}
\newcommand\ted@tl@loop{%
  \futurelet\@let@token
  \ted@tl@loop@}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@tl@loop@}
% \ldots then check what to do.
%    \begin{macrocode}
\newcommand\ted@tl@loop@{%
  \ifx\@let@token\@nil
    \let\next\ted@tl@finish
  \else
    \advance\ted@code\m@ne
    \ifnum\ted@code<\z@
      \let\next\ted@tl@finish
    \else
      \ifx\@let@token\@sptoken
        \let\next\ted@tl@space
      \else
        \let\next\ted@tl@add
      \fi
    \fi
  \fi
  \next}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@tl@add}
% \begin{macro}{\ted@tl@space}
% Normal characters are just grabbed and added without care, and spaces
% are gobbled with a special macro which also add a space to the output.
%    \begin{macrocode}
\newcommand*\ted@tl@add[1]{%
  \ted@toks\expandafter{\the\ted@toks #1}%
  \ted@tl@loop}
%    \end{macrocode}
%    \begin{macrocode}
\@ifdefinable\ted@tl@space{%
  \expandafter\def\expandafter\ted@tl@space\space{%
    \ted@tl@add{ }}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@tl@finish}
% When the end has been reached (either because a \cs{@nil} was
% encountered or because the line is almost full), it's time to
% actually display the result.  We add \cs{ETC.} at the end when the
% full \cs{meaning} isn't displayed.
%    \begin{macrocode}
\@ifdefinable\ted@tl@finish{%
  \def\ted@tl@finish#1\@nil{%
    \ifnum\ted@code<\z@
      \ted@typeout{\the\ted@toks\string\ETC.)}
    \else
      \ted@typeout{\the\ted@toks)}
    \fi}}
%    \end{macrocode}
% \end{macro}
%
% \subsection{User macros}
%
% \begin{macro}{\ted@typeout}
% Since we just discussed display, let's see the related user commands.
% Output is done with
%    \begin{macrocode}
\newcommand\ted@typeout{%
  \immediate\write\ted@outfile}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ShowTokensOnline}
% \begin{macro}{\ShowTokensLogonly}
% allowing the user to choose between online display, or log output.
% Default is online.
%    \begin{macrocode}
\newcommand\ShowTokensOnline{%
  \let\ted@outfile\@unused}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ShowTokensLogonly{%
  \let\ted@outfile\m@ne}
%    \end{macrocode}
%    \begin{macrocode}
\ShowTokensOnline
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ShowTokens}
% \begin{macro}{\ted@ShowTokens@exp}
% The user macro for showing tokens is a simple call to the internal
% macro, just expanding its argument once in its stared form.
%    \begin{macrocode}
\newcommand\ShowTokens{%
  \@ifstar{\ted@ShowTokens@exp}{\ted@ShowTokens}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@ShowTokens@exp[1]{%
  \expandafter\ted@ShowTokens\expandafter{#1}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\Substitute}
% \begin{macro}{\ted@Subs@star}
% Now, the user macro for substitution. First, check how many stars
% there are, if any, and set \cs{ted@subs@cmd} accordingly.
%    \begin{macrocode}
\newcommand\Substitute{%
  \@ifstar
  {\ted@Subs@star}
  {\let\ted@Subs@cmd\ted@Substitute \ted@Subs}}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@Subs@star{%
  \@ifstar
  {\let\ted@Subs@cmd\ted@Subs@exp@iii \ted@Subs}
  {\let\ted@Subs@cmd\ted@Subs@exp@i \ted@Subs}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@Subs@exp@i}
% \begin{macro}{\ted@Subs@exp@iii}
% Here are the intermediate macros that expand either the first or all
% three arguments before calling \cs{ted@Substitute}.
%    \begin{macrocode}
\newcommand\ted@Subs@exp@i{%
  \expandafter\ted@Substitute\expandafter}
%    \end{macrocode}
%    \begin{macrocode}
\newcommand\ted@Subs@exp@iii[3]{%
  \begingroup
  \toks0{\ted@Substitute}%
  \toks2\expandafter{#1}%
  \toks4\expandafter{#2}%
  \toks6\expandafter{#3}%
  \xdef\ted@Subs@cmd{\the\toks0{\the\toks2}{\the\toks4}{\the\toks6}}%
  \endgroup
  \ted@Subs@cmd}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \begin{macro}{\ted@Subs}
% Now, the last macro checks and process the optional argument. Here we
% set \cs{ted@output}, which will be used at the end of
% \cs{ted@Substitute}.
%    \begin{macrocode}
\newcommand\ted@Subs[1][\ted@toks]{%
  \def\ted@output{#1}%
  \ted@Subs@cmd}
%    \end{macrocode}
% \end{macro}
% \begin{macro}{\ted@output}
% Finally set a default \cs{ted@output} for advanced users who may want
% to use \cs{ted@Substitute} directly.
%    \begin{macrocode}
\let\ted@output\ted@toks
%    \end{macrocode}
% \end{macro}
%
% \bigskip
% \begin{center}\Large
% That's all folks!\\
% Happy \TeX ing!
% \end{center}
%
% \Finale
% 
% \iffalse
%</package>
%<*batchfile>
% \fi
% \begingroup\obeyspaces
% \typeout{*****************************************************}
% \typeout{*                                                   *}
% \typeout{* To finish the installation you have to move the   *}
% \typeout{* following file into a directory searched by TeX:  *}
% \typeout{*                                                   *}
% \typeout{*      ted.sty                                      *}
% \typeout{*                                                   *}
% \typeout{* Documentation is in ted.dvi or ted.pdf            *}
% \typeout{*                                                   *}
% \typeout{* To produce the french documentation, run          *}
% \typeout{*      (pdf)latex ted-fr.drv                        *}
% \typeout{*                                                   *}
% \typeout{* Happy TeXing!                                     *}
% \typeout{*                                                   *}
% \typeout{*****************************************************}
% \endgroup
% \iffalse
%</batchfile>
% \fi
\endinput