6 use CSS::Minifier::XS 0.07;
10 use File::Path qw(make_path);
14 use JavaScript::Minifier::XS;
20 my $canonical_url = 'https://www.exim.org/';
23 my %opt = parse_arguments();
25 ## setup static root location
26 ## TODO: for doc generation only this should be within the docs dir
27 $opt{staticroot} = File::Spec->catdir( $opt{docroot}, 'static' );
30 my %cache; # General cache object
31 do_doc( 'spec', $_ ) foreach @{ $opt{spec} || [] };
32 do_doc( 'filter', $_ ) foreach @{ $opt{filter} || [] };
33 do_web() if ( $opt{web} );
34 do_static() if ( $opt{web} or !$opt{localstatic} ); # need this for all other pages generated
36 ## Add the exim-html-current symlink
37 foreach my $type (qw(html pdf)) {
38 print "Symlinking exim-$type-current to exim-$type-$opt{latest}\n" if ( $opt{verbose} );
39 unlink("$opt{docroot}/exim-$type-current");
40 symlink( "exim-$type-$opt{latest}", "$opt{docroot}/exim-$type-current" )
41 || warn "symlink to $opt{docroot}/exim-$type-current failed";
44 # ------------------------------------------------------------------
45 ## Generate the website files
48 ## copy these templates to docroot...
49 copy_transform_files( "$opt{tmpl}/web", $opt{docroot}, 0 );
52 # ------------------------------------------------------------------
53 ## Generate the static file set
55 my $staticroot = shift || $opt{staticroot};
57 ## make sure I have a directory
58 mkdir($staticroot) or die "Unable to make staticroot: $!\n" unless ( -d $staticroot );
60 ## copy these templates to docroot...
61 copy_transform_files( "$opt{tmpl}/static", $staticroot, 1 );
64 # ------------------------------------------------------------------
65 ## Generate the website files
66 sub copy_transform_files {
71 ## Make sure the template web directory exists
72 die "No such directory: $source\n" unless ( -d $source );
74 ## Scan the web templates
77 my ($path) = substr( $File::Find::name, length("$source"), length($File::Find::name) ) =~ m#^/*(.*)$#;
79 if ( -d "$source/$path" ) {
81 ## Create the directory in the target if it doesn't exist
82 if ( !-d "$target/$path" ) {
83 mkdir("$target/$path") or die "Unable to make $target/$path: $!\n";
89 ## Build HTML from XSL files and simply copy static files which have changed
90 if ( ( !$static ) and ( $path =~ /(.+)\.xsl$/ ) ) {
91 print "Generating : /$1.html\n" if ( $opt{verbose} );
92 transform( undef, "$source/$path", "$target/$1.html" );
94 elsif ( -f "$source/$path" ) {
96 ## Skip if the file hasn't changed (mtime/size based)
98 if (( -f "$target/$path" )
99 and ( ( stat("$source/$path") )[9] == ( stat("$target/$path") )[9] )
100 and ( ( stat("$source/$path") )[7] == ( stat("$target/$path") )[7] ) );
102 if ( $path =~ /(.+)\.css$/ ) {
103 print "CSS to : /$path\n" if ( $opt{verbose} );
104 my $content = read_file("$source/$path");
105 write_file( "$target/$path", $opt{minify} ? CSS::Minifier::XS::minify($content) : $content );
107 elsif ( $path =~ /(.+)\.js$/ ) {
108 print "JS to : /$path\n" if ( $opt{verbose} );
109 my $content = read_file("$source/$path");
110 write_file( "$target/$path",
111 $opt{minify} ? JavaScript::Minifier::XS::minify($content) : $content );
115 print "Copying to : /$path\n" if ( $opt{verbose} );
116 copy( "$source/$path", "$target/$path" ) or die "$path: $!";
119 utime( time, ( stat("$source/$path") )[9], "$target/$path" );
128 # ------------------------------------------------------------------
129 ## Generate index/chapter files for a doc
131 my ( $type, $xml_path ) = @_;
133 ## Read and validate the XML file
134 my $xml = XML::LibXML->new(expand_entities => 1)->parse_file($xml_path) or die $!;
136 ## Get the version number
138 my $version = $xml->findvalue('/book/bookinfo/revhistory/revision/revnumber');
139 die "Unable to get version number\n"
140 unless defined $version and $version =~ /^
143 (?:\.\d+(?:\.\d+)?)? # (minor(.patch))
144 (?:\.\d+(?:\.\d+(?:\.\d+)?)?)? # (minor(.patch.(fixes)))
146 (?:-RC\d+)?$/x; # -RCX
150 ## Prepend chapter filenames?
151 my $prepend_chapter = $type eq 'filter' ? 'filter_' : '';
153 ## Add the canonical url for this document
154 $xml->documentElement()
155 ->appendTextChild( 'canonical_url',
156 "${canonical_url}exim-html-current/doc/html/spec_html/" . ( $type eq 'spec' ? 'index' : 'filter' ) . ".html" );
158 ## Add a url for the latest version of this document
159 if ( $version ne $opt{latest} ) {
160 $xml->documentElement()
161 ->appendTextChild( 'current_url',
162 "../../../../exim-html-current/doc/html/spec_html/" . ( $type eq 'spec' ? 'index' : 'filter' ) . ".html" );
166 xref_fixup( $xml, $prepend_chapter );
168 ## set the staticroot
171 ? File::Spec->catdir( $opt{docroot}, "exim-html-$version", 'doc', 'html', 'static' )
173 unless ( -d $staticroot ) {
174 make_path( $staticroot, { verbose => $opt{verbose} } );
175 do_static($staticroot);
178 ## Generate the front page
180 my $path = "exim-html-$version/doc/html/spec_html/" . ( $type eq 'filter' ? $type : 'index' ) . ".html";
181 print "Generating : docroot:/$path\n" if ( $opt{verbose} );
182 transform( $xml, "$opt{tmpl}/doc/index.xsl", "$opt{docroot}/$path", $staticroot );
185 ## Generate a Table of Contents XML file
188 "exim-html-$version/doc/html/spec_html/" . ( $type eq 'filter' ? 'filter_toc' : 'index_toc' ) . ".xml";
189 print "Generating : docroot:/$path\n" if ( $opt{verbose} );
190 transform( $xml, "$opt{tmpl}/doc/toc.xsl", "$opt{docroot}/$path", $staticroot );
193 ## Generate the chapters
195 my @chapters = map { $_->cloneNode(1) } $xml->findnodes('/book/chapter');
196 my( $chapter_title, $chapter_title_prev, $chapter_title_next );
197 foreach my $chapter (@chapters) {
199 ## Add a <chapter_id>N</chapter_id> node for the stylesheet to use
200 $chapter->appendTextChild( 'chapter_id', ++$counter );
202 ## Get the current and surrounding chapter titles
203 $chapter_title_prev = $chapter_title;
204 $chapter_title = $chapter_title_next || $chapter->findvalue('title_uri');
205 $chapter_title_next = $chapters[$counter]->findvalue('title_uri') if $counter < int(@chapters);
207 ## Add previous/next/canonical urls for nav
209 $chapter->appendTextChild( 'prev_url',
214 : sprintf( '%sch-%s.html', $prepend_chapter, $chapter_title_prev ) );
215 $chapter->appendTextChild( 'this_url', sprintf( '%sch-%s.html', $prepend_chapter, $chapter_title ) );
216 $chapter->appendTextChild( 'next_url', sprintf( '%sch-%s.html', $prepend_chapter, $chapter_title_next ) )
217 unless int(@chapters) == $counter;
218 $chapter->appendTextChild( 'toc_url', ( $type eq 'filter' ? 'filter' : 'index' ) . '.html' );
219 $chapter->appendTextChild(
222 'https://www.exim.org/exim-html-current/doc/html/spec_html/%sch-%s.html',
223 $prepend_chapter, $chapter_title
226 if ( $version ne $opt{latest} ) {
227 $chapter->appendTextChild(
230 '../../../../exim-html-current/doc/html/spec_html/%sch-%s.html',
231 $prepend_chapter, $chapter_title
237 ## Create an XML document from the chapter
238 my $doc = XML::LibXML::Document->createDocument( '1.0', 'UTF-8' );
239 $doc->setDocumentElement($chapter);
241 ## Transform the chapter into html
243 my $real_path = sprintf( 'exim-html-%s/doc/html/spec_html/%sch-%s.html', $version, $prepend_chapter, $chapter_title );
244 my $link_path = sprintf( 'exim-html-%s/doc/html/spec_html/%sch%02d.html', $version, $prepend_chapter, $counter );
245 print "Generating : docroot:/$real_path\n" if ( $opt{verbose} );
246 transform( $doc, "$opt{tmpl}/doc/chapter.xsl", "$opt{docroot}/$real_path", $staticroot );
247 # Making a relative symlink to a file in the same directory.
248 # Extract just the filename portion of $real_path.
249 my $real_file = basename($real_path);
250 print "Symlinking : docroot:/$link_path to $real_file\n" if ( $opt{verbose} );
251 if ( -f "$opt{docroot}/$link_path" ) {
252 unlink("$opt{docroot}/$link_path") or die "failed removing $opt{docroot}/$link_path: $!";
254 symlink( "$real_file", "$opt{docroot}/$link_path" ) || die "symlink to $opt{docroot}/$link_path failed: $!";
259 # ------------------------------------------------------------------
262 my ( $xml, $prepend_chapter ) = @_;
266 ## Add the "prepend_chapter" info
267 ( $xml->findnodes('/book') )[0]->appendTextChild( 'prepend_chapter', $prepend_chapter );
269 ## Iterate over each chapter
270 my $chapter_counter = 0;
271 foreach my $chapter ( $xml->findnodes('/book/chapter') ) {
274 my $chapter_id = $chapter->getAttribute('id');
275 unless ($chapter_id) { # synthesise missing id
276 $chapter_id = sprintf( 'chapter_noid_%04d', $chapter_counter );
277 $chapter->setAttribute( 'id', $chapter_id );
279 my $chapter_title = $chapter->findvalue('title');
281 ## Set title_uri so we can use eg ch-introduction.html instead of ch01.html
282 $chapter->appendTextChild( 'title_uri', title_to_uri($chapter_title) );
284 $index{$chapter_id} = { chapter_id => $chapter_counter, chapter_title => $chapter_title };
286 ## Iterate over each section
287 my $section_counter = 0;
288 foreach my $section ( $chapter->findnodes('section') ) {
290 $section->setAttribute( 'sectprefix', $section_counter );
292 my $section_id = $section->getAttribute('id');
293 unless ($section_id) { # synthesise missing id
294 $section_id = sprintf( 'section_noid_%04d_%04d', $chapter_counter, $section_counter );
295 $section->setAttribute( 'id', $section_id );
297 my $section_title = $section->findvalue('title');
299 $index{$section_id} = {
300 chapter_id => $chapter_counter,
301 chapter_title => $chapter_title,
302 section_id => $section_counter,
303 section_title => $section_title
306 # 2022/07/07 jgh: added loop for sections under sections, which are resulting from the .subsection macro
307 # Add a "level" attribute to these nodes
308 ## Iterate over each subsection
309 my $subsec_counter = 0;
310 foreach my $subsection ( $section->findnodes('section') ) {
313 $subsection->setAttribute( 'level', "2" );
314 $subsection->setAttribute( 'sectprefix', sprintf("%d.%d", $section_counter, $subsec_counter) );
316 my $subsec_id = $subsection->getAttribute('id');
317 unless ($subsec_id) { # synthesise missing id
318 $subsec_id = sprintf( 'section_noid_%04d_%04d_%04d', $chapter_counter, $section_counter, $subsec_counter );
319 $subsection->setAttribute( 'id', $subsec_id );
321 my $subsec_title = $subsection->findvalue('title');
323 $index{$subsec_id} = {
324 chapter_id => $chapter_counter,
325 chapter_title => $chapter_title,
326 section_id => $subsec_counter,
327 section_title => $subsec_title
332 ## Build indexes as new chapters
333 build_indexes( $xml, $prepend_chapter, \%index );
335 ## Replace all of the xrefs in the XML
336 foreach my $xref ( $xml->findnodes('//xref') ) {
337 my $linkend = $xref->getAttribute('linkend');
339 if ( exists $index{$linkend} ) {
340 $xref->setAttribute( 'chapter_id', $index{$linkend}{'chapter_id'} ) if ( $index{$linkend}{'chapter_id'} );
341 $xref->setAttribute( 'chapter_title', $index{$linkend}{'chapter_title'} );
342 $xref->setAttribute( 'section_id', $index{$linkend}{'section_id'} ) if ( $index{$linkend}{'section_id'} );
343 $xref->setAttribute( 'section_title', $index{$linkend}{'section_title'} )
344 if ( $index{$linkend}{'section_title'} );
345 $xref->setAttribute( 'url',
346 sprintf( '%sch-%s.html', $prepend_chapter, title_to_uri($index{$linkend}{'chapter_title'}) )
347 . ( $index{$linkend}{'section_id'} ? '#' . $linkend : '' ) );
352 # ------------------------------------------------------------------
355 my ( $xml, $prepend_chapter, $xref ) = @_;
358 my $seealso_hash = {};
360 my $verterm_counter = 0;
362 foreach my $node ( $xml->findnodes('//section | //chapter | //varlistentry | //indexterm') ) {
363 if ( $node->nodeName eq 'indexterm' ) {
364 my $role = $node->getAttribute('role') || 'concept';
365 my $primary = $node->findvalue('child::primary');
366 my $first = ( $primary =~ /^[A-Za-z]/ ) ? uc( substr( $primary, 0, 1 ) ) : ''; # first letter or marker
367 my $secondary = $node->findvalue('child::secondary') || '';
368 my $see = $node->findvalue('child::see');
369 my $see_also = $node->findvalue('child::seealso');
371 next unless ( $primary || $secondary ); # skip blank entries for now...
373 $index_hash->{$role}{$first}{$primary}{$secondary} ||= [];
374 if ( $see || $see_also ) {
375 # The scalar value being written here assumes only one seealso on an indeed term
376 # It would be nice to have the $see displayed in bold rather than in quotes
377 $seealso_hash->{$role}{$first}{$primary}{$secondary} = 'see "' . $see .'"' if ($see);
378 $seealso_hash->{$role}{$first}{$primary}{$secondary} = 'see also "' . $see_also .'"' if ($see_also);
382 push @{ $index_hash->{$role}{$first}{$primary}{$secondary} }, $current_id;
385 elsif ( $node->nodeName eq 'varlistentry' ) {
387 foreach my $vitem ( $node->findnodes('listitem') ) {
389 # Add an anchorname xml attribute.
390 # chapter.xsl spots this and places a "<a id="{@anchorname}"> </a>"
392 my $anchorname = sprintf("vi%d", $verterm_counter++);
393 $vitem->setAttribute( 'anchorname', $anchorname );
394 $current_id = $anchorname;
396 # Set the latest indexable id to be picked up by the next indexterm,
397 # which should be in the content of the listitem
399 my ($chapter_title, $sec_id, $sec_title);
401 foreach my $chap ( $node->findnodes('ancestor::chapter') ) {
402 $chapter_title = $chap->findvalue('title');
404 next unless ($chapter_title);
406 # Search upward to find a subsection or section id & title
407 foreach my $ssec ( $node->findnodes("ancestor::section[\@level='2']") ) {
408 $sec_id = $ssec->getAttribute('id');
409 $sec_title = $ssec->findvalue('title');
412 if (!defined($sec_id)) {
413 foreach my $sec ( $node->findnodes('ancestor::section') ) {
414 $sec_id = $sec->getAttribute('id');
415 $sec_title = $sec->findvalue('title');
420 $xref->{$anchorname}{'chapter_title'} = $chapter_title;
421 $xref->{$anchorname}{'section_id'} = $sec_id if ($sec_id);
422 $xref->{$anchorname}{'section_title'} = $sec_title if ($sec_title);
426 $current_id = $node->getAttribute('id');
430 # now we build a set of new chapters with the index data in
431 my $book = ( $xml->findnodes('/book') )[0];
432 foreach my $role ( sort { $a cmp $b } keys %{$index_hash} ) {
433 my $chapter = XML::LibXML::Element->new('chapter');
434 $book->appendChild($chapter);
435 $chapter->setAttribute( 'id', join( '_', 'index', $role ) );
436 $chapter->setAttribute( 'class', 'index' );
437 $chapter->appendTextChild( 'title', ( ucfirst($role) . ' Index' ) );
438 $chapter->appendTextChild( 'title_uri', title_to_uri(ucfirst($role) . ' Index') );
440 foreach my $first ( sort { $a cmp $b } keys %{ $index_hash->{$role} } ) {
441 my $section = XML::LibXML::Element->new('section');
442 my $list = XML::LibXML::Element->new('variablelist');
443 $chapter->appendChild($section);
444 $section->setAttribute( 'id', join( '_', 'index', $role, $first ) );
445 $section->setAttribute( 'class', 'index' );
446 $section->appendTextChild( 'title', $first ? $first : 'Symbols' );
447 $section->appendChild($list);
448 foreach my $primary ( sort { $a cmp $b } keys %{ $index_hash->{$role}{$first} } ) {
449 my $entry = XML::LibXML::Element->new('varlistentry');
450 my $item = XML::LibXML::Element->new('listitem');
451 $list->appendChild($entry)->appendTextChild( 'term', $primary );
452 $entry->appendChild($item);
454 foreach my $secondary ( sort { $a cmp $b } keys %{ $index_hash->{$role}{$first}{$primary} } ) {
455 my $para = XML::LibXML::Element->new('para');
456 if ( $secondary eq '' ) {
457 $item->appendChild($para); # skip having extra layer of heirarchy
461 $slist = XML::LibXML::Element->new('variablelist');
462 $item->appendChild($slist);
464 my $sentry = XML::LibXML::Element->new('varlistentry');
465 my $sitem = XML::LibXML::Element->new('listitem');
466 $slist->appendChild($sentry)->appendTextChild( 'term', $secondary );
467 $sentry->appendChild($sitem)->appendChild($para);
470 my $seealso = $seealso_hash->{$role}{$first}{$primary}{$secondary};
471 $para->appendText($seealso) if ($seealso);
474 foreach my $ref ( @{ $index_hash->{$role}{$first}{$primary}{$secondary} } ) {
475 $para->appendText(', ')
477 my $xrefel = XML::LibXML::Element->new('xref');
478 $xrefel->setAttribute( linkend => $ref );
479 $xrefel->setAttribute( longref => 1 );
480 $para->appendChild($xrefel);
488 # ------------------------------------------------------------------
489 ## Handle the transformation
491 my ( $xml, $xsl_path, $out_path, $staticroot_abs ) = @_;
493 ## make sure $staticroot is set
494 $staticroot_abs ||= $opt{staticroot};
496 ## Build an empty XML structure if an undefined $xml was passed
497 unless ( defined $xml ) {
498 $xml = XML::LibXML::Document->createDocument( '1.0', 'UTF-8' );
499 $xml->setDocumentElement( $xml->createElement('content') );
502 ## Add the current version of Exim to the XML
503 $xml->documentElement()->appendTextChild( 'current_version', $opt{latest} );
505 ## Add the old versions of Exim to the XML
506 $xml->documentElement()->appendTextChild( 'old_versions', $_ ) foreach old_docs_versions();
508 ## Parse the ".xsl" file as XML
509 my $xsl = XML::LibXML->new(expand_entities => 1)->parse_file($xsl_path) or die $!;
511 ## Generate a stylesheet from the ".xsl" XML.
512 my $stylesheet = XML::LibXSLT->new()->parse_stylesheet($xsl);
514 ## work out the static root relative to the target
515 my $target_dir = ( File::Spec->splitpath($out_path) )[1];
516 my $staticroot = File::Spec->abs2rel( $staticroot_abs, $target_dir );
518 ## Generate a doc from the XML transformed with the XSL
519 my $doc = $stylesheet->transform( $xml, staticroot => sprintf( "'%s'", $staticroot ) );
521 ## Make the containing directory if it doesn't exist
522 make_path( ( $out_path =~ /^(.+)\/.+$/ )[0], { verbose => $opt{verbose} } );
524 ## Write out the document
525 open my $out, '>', $out_path or die "Unable to write $out_path - $!";
526 print $out $stylesheet->output_as_bytes($doc);
530 # ------------------------------------------------------------------
531 ## Takes a chapter title and fixes it up so it is suitable for use in a URI
533 my $title = lc(shift);
534 $title =~ s/[^a-z0-9\s]+//gi; # Only allow spaces, numbers and letters
535 $title =~ s/\s+/_/g; # Replace spaces with underscores so URLs are easier to copy about
539 # ------------------------------------------------------------------
540 ## Look in the docroot for old versions of the documentation
541 sub old_docs_versions {
542 if ( !exists $cache{old_docs_versions} ) {
544 foreach ( glob("$opt{docroot}/exim-html-*") ) {
545 push @versions, $1 if /-(\d+(?:\.\d+)?)$/ && $1 lt $opt{latest} && -d $_;
547 $cache{old_docs_versions} = [ reverse sort { $a cmp $b } @versions ];
549 return @{ $cache{old_docs_versions} };
552 # ------------------------------------------------------------------
558 pod2usage( -exitval => 1, -verbose => 0 );
561 # ------------------------------------------------------------------
563 sub parse_arguments {
565 my %opt = ( spec => [], filter => [], help => 0, man => 0, web => 0, minify => 1, verbose => 0, localstatic => 0, tmpl => "$Bin/../templates" );
567 \%opt, 'help|h!', 'man!', 'web!', 'spec=s{1,}', 'filter=s{1,}',
568 'latest=s', 'tmpl=s', 'docroot=s', 'minify!', 'verbose!', 'localstatic!'
569 ) || pod2usage( -exitval => 1, -verbose => 0 );
572 pod2usage(0) if ( $opt{help} );
573 pod2usage( -verbose => 2 ) if ( $opt{man} );
575 ## --spec and --filter lists
576 foreach my $set (qw[spec filter]) {
578 [ map { my $f = File::Spec->rel2abs($_); error_help( 'No such file: ' . $_ ) unless -f $f; $f }
582 error_help('Missing value for latest') unless ( exists( $opt{latest} ) && defined( $opt{latest} ) );
583 error_help('Invalid value for latest') unless $opt{latest} =~ /^\d+(?:\.\d+)*$/;
585 ## --tmpl and --docroot
586 foreach my $set (qw[tmpl docroot]) {
587 error_help( 'Missing value for ' . $set ) unless ( exists( $opt{$set} ) && defined( $opt{$set} ) );
588 my $f = File::Spec->rel2abs( $opt{$set} );
589 error_help( 'No such directory: ' . $opt{$set} ) unless -d $f;
592 error_help('Excess arguments') if ( scalar(@ARGV) );
594 error_help('Must include at least one of --web, --spec or --filter')
595 unless ( $opt{web} || scalar( @{ $opt{spec} || [] } ) || scalar( @{ $opt{filter} || [] } ) );
600 # ------------------------------------------------------------------
607 gen - Generate exim html documentation and website
614 --help display this help and exits
615 --man displays man page
616 --spec file... spec docbook/XML source files
617 --filter file... filter docbook/XML source files
618 --web Generate the general website pages
619 --latest VERSION Required. Specify the latest stable version of Exim.
620 --tmpl PATH Required. Path to the templates directory
621 --docroot PATH Required. Path to the website document root
622 --[no-]minify [Don't] minify CSS and Javascript
623 --localstatic Makes the static files local to each doc ver
631 Display help and exits
637 =item B<--spec> I<file...>
639 List of files that make up the specification documentation docbook/XML source
642 =item B<--filter> I<file...>
644 List of files that make up the filter documentation docbook/XML source files.
648 Generate the website from the template files.
650 =item B<--latest> I<version>
652 Specify the current exim version. This is used to create links to the current
655 This option is I<required>
657 =item B<--tmpl> I<directory>
659 Specify the directory that the templates are kept in.
661 This option is I<required>
663 =item B<--docroot> I<directory>
665 Specify the directory that the output should be generated into. This is the
666 website C<docroot> directory.
668 This option is I<required>
672 If this option is set then both the CSS and Javascript files processed are
673 minified using L<CSS::Minifier::XS> and L<JavaScript::Minifier::XS>
676 This option is set by default - to disable it specify C<--no-minify>
678 =item B<--localstatic>
680 Makes the static files (CSS, images etc), local for each version of the
681 documentation. This is more suitable for packaged HTML documentation.
687 Generates the exim website and HTML documentation.
693 --spec docbook/*/spec.xml \
694 --filter docbook/*/filter.xml \
697 --docroot /tmp/website
703 Nigel Metheringham <nigel@exim.org> - mostly broke the framework Mike produced.
707 Copyright 2010-2012 Exim Maintainers. All rights reserved.