Convert SSI includes to tt includes in manpages

This was a bigger effort, and involves a new target, manhtml.

Reviewed-by: Anton Arapov <[email protected]>
Reviewed-by: Tomas Vavra <[email protected]>
(Merged from openssl#430)

Author: levitte

Makefile

@@ -128,13 +128,13 @@ TECHNICAL_POLICIES=$(filter-out $(CHECKOUTS)/technical-policies/policies/README.
 	@rm -f $@
 	./bin/from-tt $<
 
-all: suball subdocs manmaster mancross sitemap akamai-purge
+all: suball subdocs manmastertts mancross manhtml sitemap akamai-purge
 
 suball: $(SIMPLE) $(SRCLISTS)
 
 relupd: suball docs sitemap akamai-purge
 
-docs: subdocs manpages mancross
+docs: subdocs manpagetts mancross manhtml
 
 subdocs: $(SIMPLEDOCS)
 
@@ -171,9 +171,9 @@ bin/md-to-html5: inc/pandoc-template.html5
 ##
 ##  A lot of the work is made with generated rules.
 
-# makemanpages1 and makemanpages3 creates rules for targets like man-pages-1.1.1,
-# to build the set of man-pages.  makemanpages1 is used for pre-3.0 OpenSSL,
-# while makemanpages3 is used for OpenSSL 3.0 and on.
+# makemanpagetts1 and makemanpagetts3 creates rules for targets like man-pagetts-1.1.1,
+# to build the set of man-page templates.  makemanpagetts1 is used for pre-3.0 OpenSSL,
+# while makemanpagetts3 is used for OpenSSL 3.0 and on.
 # makemanapropos creates rules for targets like man-apropos-1.1.1, to build
 # 'apropos' like indexes for all the manpages.
 # makemanindexes creates rules for targets like man-index-1.1.1, to build the
@@ -182,26 +182,26 @@ bin/md-to-html5: inc/pandoc-template.html5
 # $(1) = input directory in CHECKOUTS, $(2) = release version
 
 # This variant is for pre-3.0 documentation
-define makemanpages1
-man-pages-$(2):
+define makemanpagetts1
+man-pagetts-$(2):
 	@rm -rf docs/man$(2)
 	@mkdir -p docs/man$(2) \
 		  docs/man$(2)/man1 \
 		  docs/man$(2)/man3 \
 		  docs/man$(2)/man5 \
 		  docs/man$(2)/man7
-	./bin/mk-manpages $(CHECKOUTS)/$(1)/doc $(2) docs/man$(2)
+	./bin/mk-manpagetts $(CHECKOUTS)/$(1)/doc $(2) docs/man$(2)
 endef
 # This variant is for 3.0 documentation
-define makemanpages3
-man-pages-$(2):
+define makemanpagetts3
+man-pagetts-$(2):
 	@rm -rf docs/man$(2)
 	@mkdir -p docs/man$(2) \
 		  docs/man$(2)/man1 \
 		  docs/man$(2)/man3 \
 		  docs/man$(2)/man5 \
 		  docs/man$(2)/man7
-	./bin/mk-manpages3 $(CHECKOUTS)/$(1) $(2) docs/man$(2)
+	./bin/mk-manpagetts3 $(CHECKOUTS)/$(1) $(2) docs/man$(2)
 endef
 define makemanapropos
 docs/man$(2)/man1/index.inc: bin/mk-apropos Makefile
@@ -246,33 +246,33 @@ endef
 define makemandirdata
 docs/man$(2)/man1/dirdata.yaml: docs/sub-dirdata.yaml.tt bin/from-tt Makefile
 	./bin/from-tt -d docs/man$(2)/man1 \
-		      releases='$(MANSERIES)' release='$(2)' \
+		      releases='$(MANSERIES)' release='$(2)' sectnum=1 \
 		      < $$< > $$@
 docs/man$(2)/man3/dirdata.yaml: docs/sub-dirdata.yaml.tt bin/from-tt Makefile
 	./bin/from-tt -d docs/man$(2)/man3 \
-		      releases='$(MANSERIES)' release='$(2)' \
+		      releases='$(MANSERIES)' release='$(2)' sectnum=3 \
 		      < $$< > $$@
 docs/man$(2)/man5/dirdata.yaml: docs/sub-dirdata.yaml.tt bin/from-tt Makefile
 	./bin/from-tt -d docs/man$(2)/man5 \
-		      releases='$(MANSERIES)' release='$(2)' \
+		      releases='$(MANSERIES)' release='$(2)' sectnum=5 \
 		      < $$< > $$@
 docs/man$(2)/man7/dirdata.yaml: docs/sub-dirdata.yaml.tt bin/from-tt Makefile
 	./bin/from-tt -d docs/man$(2)/man7 \
-		      releases='$(MANSERIES)' release='$(2)' \
+		      releases='$(MANSERIES)' release='$(2)' sectnum=7 \
 		      < $$< > $$@
 docs/man$(2)/dirdata.yaml: docs/sub-dirdata.yaml.tt bin/from-tt Makefile
 	./bin/from-tt -d docs/man$(2) \
                       releases='$(MANSERIES)' release='$(2)' \
 		      < $$< > $$@
 endef
 define makemanuals1
-$(eval $(call makemanpages1,$(1),$(2)))
+$(eval $(call makemanpagetts1,$(1),$(2)))
 $(eval $(call makemanapropos,$(1),$(2)))
 $(eval $(call makemanindexes,$(1),$(2)))
 $(eval $(call makemandirdata,$(1),$(2)))
 endef
 define makemanuals3
-$(eval $(call makemanpages3,$(1),$(2)))
+$(eval $(call makemanpagetts3,$(1),$(2)))
 $(eval $(call makemanapropos,$(1),$(2)))
 $(eval $(call makemanindexes,$(1),$(2)))
 $(eval $(call makemandirdata,$(1),$(2)))
@@ -292,19 +292,40 @@ $(foreach S,$(MANSERIES3),$(eval $(call makemanuals3,openssl-$(S),$(S))))
 # source from $(CHECKOUTS)/openssl-x.y.z-stable/doc
 $(foreach S,$(MANSERIES1),$(eval $(call makemanuals1,openssl-$(S)-stable,$(S))))
 
-MANMASTER_TARGETS = \
-        man-pages-master docs/manmaster/index.html \
-        $(foreach SEC,1 3 5 7, docs/manmaster/man$(SEC)/index.html)
-manmaster: $(MANMASTER_TARGETS)
-MANPAGES_TARGETS = \
+MANMASTER_DIRS = \
+	$(foreach SEC,1 3 5 7, docs/manmaster/man$(SEC))
+MANMASTERTT_TARGETS = \
+        man-pagetts-master \
+        docs/manmaster/dirdata.yaml docs/manmaster/index.html \
+        $(foreach D,$(MANMASTER_DIRS), $(D)/dirdata.yaml $(D)/index.html)
+manmastertts: $(MANMASTERTT_TARGETS)
+MANPAGE_DIRS = \
         $(foreach S,$(MANSERIES), \
-          man-pages-$(S) docs/man$(S)/index.html \
-          $(foreach SEC,1 3 5 7, docs/man$(S)/man$(SEC)/index.html))
-manpages: manmaster $(MANPAGES_TARGETS)
+          $(foreach SEC,1 3 5 7, docs/man$(S)/man$(SEC)))
+MANPAGETT_TARGETS = \
+        $(foreach S,$(MANSERIES), man-pagetts-$(S) \
+          docs/man$(S)/dirdata.yaml docs/man$(S)/index.html) \
+        $(foreach D,$(MANPAGE_DIRS), $(D)/dirdata.yaml $(D)/index.html)
+manpagetts: manmastertts $(MANPAGETT_TARGETS)
 
 mancross:
 	./bin/mk-mancross master $(MANSERIES)
 
+# We can't get all the files when make is started, but we can make the
+# patterns for a for loop.
+MANHTML_TTPATTERNS=$(foreach D,$(MANMASTER_DIRS) $(MANPAGE_DIRS), $(D)/*.md.tt)
+manhtml:
+	@set -e; for t in $(MANHTML_TTPATTERNS); do \
+		if ! [ -f "$$t" ]; then continue; fi; \
+		d="$$(dirname $$t)"; \
+		h="$$(basename "$$t" .md.tt)"; \
+		i=; \
+		if [ "$$h" = "index" ]; then i=" -i"; fi; \
+		echo "$$t -> $$h.html"; \
+		./bin/from-tt -d "$$d" < "$$d/$$h.md.tt" \
+			| ./bin/md-to-html5$$i -o "$$d/$$h.html"; \
+	done
+
 docs/manpages.md: docs/manpages.md.tt Makefile bin/from-tt
 	@rm -f $@
 	./bin/from-tt releases='master $(MANSERIES)' $<

bin/mk-apropos

@@ -4,15 +4,15 @@
 dir=$1
 cd $dir
 
-for m in `find . -name '*.html' | sort`; do
-    description=`grep -F '<!-- OSSL: description:' $m | sed -e 's|^[^:]*: *||' -e 's|^[^:]*: *||' -e 's| *-->||'`
+for m in `find . -name '*.md.tt' | sort`; do
+    description=`grep '^OSSL-description:' $m | sed -e 's|^[^:]*: *||' -e 's|^ *"||' -e 's|" *||'`
     # If there isn't a description, it isn't a manpage and should not be
     # included
     if [ "$description" = "" ]; then
 	continue
     fi
     manfile=`echo $m | sed -e 's|\./||'`
-    manname=`basename $manfile .html`
+    manname=`basename $manfile .md.tt`
     origmanfile=`echo $manfile | sed -e "s|^$subdir|$origsubdir|"`
-    echo "<tr><td><a href=\"$manfile\">$manname</a></td><td>$description</td></tr>"
+    echo "| [$manname]($manname.html) | $description |"
 done

bin/mk-mancross

@@ -21,39 +21,31 @@ for r in $releases; do
     echo >&2 -n "$r : "
     for s in 1 3 5 7; do
 	echo >&2 -n $s
-	for f in docs/man$r/man$s/*.html; do
+	for f in docs/man$r/man$s/*.md.tt; do
 	    $debug -n '.'
-	    # Check that it's a rendered manpage
-	    if grep -F '<!-- OSSL: description:' $f > /dev/null; then
-		existsin=''
-		b=`basename $f .html`
-		for x in $releases; do
-		    if [ "$x" = "$r" ]; then
-			continue
-		    fi
-		    if [ -f "docs/man$x/man$s/$b.html" ]; then
-			existsin="$existsin $x"
-		    fi
-		done
-		(
-		    if [ "$existsin" != "" ]; then
-			cat <<EOF
-  <section>
-    <h1>This manpage</h1>
-    <ul>
-EOF
-			for x in $existsin; do
-			    cat <<EOF
-      <li><a href="/docs/man$x/man$s/$b.html">$x version</a></li>
+	    existsin=''
+	    b=`basename $f .md.tt`
+	    for x in $releases; do
+		if [ "$x" = "$r" ]; then
+		    continue
+		fi
+		if [ -f "docs/man$x/man$s/$b.md.tt" ]; then
+		    existsin="$existsin $x"
+		fi
+	    done
+	    (
+		if [ "$existsin" != "" ]; then
+		    cat <<EOF
+sidebar: |
+  # This manpage
 EOF
-			done
+		    for x in $existsin; do
 			cat <<EOF
-    </ul>
-  </section>
+  -   [$x version]([% top %]docs/man$x/man$s/$b.html)
 EOF
-		    fi
-		) > docs/man$r/man$s/$b.cross
-	    fi
+		    done
+		fi
+	    ) > docs/man$r/man$s/$b.cross
 	done
     done
     echo >&2

bin/mk-manpages bin/mk-manpagettsbin/mk-manpages renamed to bin/mk-manpagetts

@@ -64,83 +64,49 @@ sub main {
                 print $fh $out or $class->die("Can't print $outinc: $!");
                 close($fh) or $class->die("Can't close $outinc: $!");
 
-                my @htmlnames =
+                my @mdttnames =
                     map { (my $x = $_) =~ s|/|-|g; $x } @{$data{names}};
                 # Older OpenSSL pods have file names that do not correspond
                 # to any of the names in the NAME section.
                 # Strictly speaking, we shouldn't use that name, but HTML
                 # pages with that name have been produced in the past, so
                 # we keep doing so as long as it's relevant.
-                if (! grep { $_ eq $origbase } @htmlnames) {
-                    push @htmlnames, $origbase;
+                if (! grep { $_ eq $origbase } @mdttnames) {
+                    push @mdttnames, $origbase;
                 }
-                foreach my $htmlname (@htmlnames) {
-                    my $htmlfile = File::Spec->catdir( "man$data{sectnum}",
-                                                       "$htmlname.html" );
-                    my $outhtml = File::Spec->catfile( $wwwdir, $htmlfile );
-                    $out = $class->genhtml( $release, $title, $origbase,
-                                            $htmlname, %data );
-                    open( $fh, ">", $outhtml )
-                        or $class->die("Can't open $outhtml: $!");
-                    print $fh $out or $class->die("Can't print $outhtml: $!");
-                    close($fh) or $class->die("Can't close $outhtml: $!");
+                foreach my $mdttname (@mdttnames) {
+                    my $mdttfile = File::Spec->catdir( "man$data{sectnum}",
+                                                       "$mdttname.md.tt" );
+                    my $outmdtt = File::Spec->catfile( $wwwdir, $mdttfile );
+                    $out = $class->genmdtt( $release, $title, $origbase,
+                                            $mdttname, %data );
+                    open( $fh, ">", $outmdtt )
+                        or $class->die("Can't open $outmdtt: $!");
+                    print $fh $out or $class->die("Can't print $outmdtt: $!");
+                    close($fh) or $class->die("Can't close $outmdtt: $!");
                 }
             }
         }
     }
 }
 
-# Generate manpag HTML wrapper
-sub genhtml {
-    my ( $class, $release, $title, $origbase, $htmlbase, %data ) = @_;
+# Generate manpage markdown template wrapper
+sub genmdtt {
+    my ( $class, $release, $title, $origbase, $mdttbase, %data ) = @_;
+    # Note that the .inc file is pure HTML.
+    # That's OK, markdown can include HTML, by definition.
     return <<EOH;
-<!DOCTYPE html>
-<html lang="en">
-<!-- OSSL: original subdir: $data{subdir} -->
-<!-- OSSL: subdir: man$data{sectnum} -->
-<!-- OSSL: section: $data{sectnum} -->
-<!-- OSSL: description: $data{description} -->
-<!--#include virtual="/inc/head.shtml" -->
-<body>
-<!--#include virtual="/inc/banner.shtml" -->
-  <div id="main">
-    <div id="content">
-      <div class="blog-index">
-        <article>
-          <header><h2>$title</h2></header>
-          <div class="entry-content">
-            <p>
-<!--#include virtual="$origbase.inc" -->
-            </p>
-          </div>
-          <footer>
-            You are here: <a href="/">Home</a>
-            : <a href="/docs">Docs</a>
-            : <a href="/docs/manpages.html">Manpages</a>
-            : <a href="/docs/man$release/">$release</a>
-            : <a href="/docs/man$release/man$data{sectnum}">man$data{sectnum}</a>
-            : <a href="/docs/man$release/man$data{sectnum}/$htmlbase.html">$htmlbase</a>
-            <br/><a href="/sitemap.txt">Sitemap</a>
-          </footer>
-        </article>
-      </div>
-      <aside class="sidebar">
-        <section>
-          <h1><a href="/docs/man$release/">$release manpages</a></h1>
-          <ul>
-            <li><a href="../man1">Commands</a></li>
-            <li><a href="../man3">Libraries</a></li>
-            <li><a href="../man5">File Formats</a></li>
-            <li><a href="../man7">Overviews</a></li>
-          </ul>
-        </section>
-<!--#include virtual="$htmlbase.cross" -->
-      </aside>
-    </div>
-  </div>
-<!--#include virtual="/inc/footer.shtml" -->
-</body>
-</html>
+---
+OSSL-original-subdir: $data{subdir}
+OSSL-subdir: man$data{sectnum}
+OSSL-section: $data{sectnum}
+OSSL-description: "$data{description}"
+breadcrumb: $mdttbase
+[% INCLUDE "$mdttbase.cross" %]
+---
+## $title
+
+[% INCLUDE "$origbase.inc" %]
 EOH
 }
 
@@ -263,34 +229,44 @@ __END__
 
 =head1 NAME
 
-mk-manpages - htmlize man pages from POD for the OpenSSL website
+mk-manpagetts - man pages from POD into markdown/html templates
 
 =head1 SYNOPSIS
 
-mk-manpages [options] <SrcDir> <RelVer> <WwwDir>
+ mk-manpagetts [options] <SrcDir> <RelVer> <WwwDir>
 
   <SrcDir>   doc directory of release <RelVer>, example 'OpenSSL_1_0_2-stable/doc'
   <RelVer>   version number associated with <SrcDir>, example '1.0.2'
-  <WwwDir>   top level directory beneath which generated html is stored, example 'web'
+  <WwwDir>   top level directory beneath which generated template is stored,
+             example 'web'
 
     --help    display a brief help message
     --man     display full documentation
 
+This script is only useful with pre-3.0 OpenSSL.  For OpenSSL 3.0 and on,
+please use mk-manpagetts3.
+
 =head1 DESCRIPTION
 
-This utility is run on a web server generate the htmlized version of
-OpenSSL documentation from the original POD.  The resultant directory
+This utility is run on a web server generate the markdown/html template version
+of OpenSSL documentation from the original POD.  The resultant directory
 structure may look something like the following (where the contents of
 index.html do not come from this tool):
 
  $ ls some/path/to/web
  man1.0.2    man1.1.0    manmaster
  $ ls some/path/to/web/man1.0.2
- apps        crypto      index.html  ssl
+ index.html  man1        man3        man5        man7
  $ ls some/path/to/web/man1.0.2/apps
- CA.pl.html
- asn1parse.html
- c_rehash.html
+ CA.pl.md.tt
+ asn1parse.md.tt
+ c_rehash.md.tt
  ...
 
+The result needs further processing by other tools.  Among others, the .md.tt
+files rely on the presence of a corresponding .cross file, which is usually
+produced later, when all .md.tt files are in place.  To finish everything off,
+the .md.tt files need to be passed through C<tpage>, and the resulting markdown
+must be converted into pure html with a well chosen tool.
+
 =cut