diff --git a/.gitignore b/.gitignore index 3f9d398..ff1ce00 100644 --- a/.gitignore +++ b/.gitignore @@ -11,9 +11,12 @@ *.synctex.gz *.DS_Store *.xwm +*.mw datetime-defaults.sty datetime.sty check-openpmix figs/~$drawings.pptx *.loc *.soc +_minted* +sources/_autogen_/ diff --git a/Makefile b/Makefile index 32ff666..8c4f896 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Makefile for the PMIx Standard document in LaTex format. # For more information, see the master document, pmix-standard.tex. +LATEX_C=pdflatex -shell-escape -file-line-error + version=4.0 default: pmix-standard.pdf @@ -27,11 +29,8 @@ CHAPTERS= \ App_Python.tex \ Acknowledgements.tex -SOURCES= -# SOURCES=sources/*.c \ -# sources/*.cpp \ -# sources/*.f90 \ -# sources/*.f +SOURCES=sources/*.c \ + sources/*.py \ INTERMEDIATE_FILES=pmix-standard.pdf \ pmix-standard.toc \ @@ -45,27 +44,34 @@ INTERMEDIATE_FILES=pmix-standard.pdf \ pmix-standard.blg \ pmix-standard.synctex.gz \ pmix-standard.xwm \ - *.idx *.ilg *.ind + pmix-standard.mw \ + pmix-standard.loc \ + pmix-standard.soc \ + *.idx *.ilg *.ind \ + _minted-* \ + sources/_autogen_ all: pmix-standard.pdf pmix-standard.pdf: $(CHAPTERS) $(SOURCES) pmix.sty pmix-standard.tex figs/pmix-logo.png - rm -f $(INTERMEDIATE_FILES) + rm -rf $(INTERMEDIATE_FILES) @echo "-------------------------------------------------------------" @echo "If error occurs check pmix-standard.log and pmix-standard.ind" @echo "-------------------------------------------------------------" + @echo "====> Preprocess Examples" + @./bin/process-example.py $(SOURCES) @echo "====> Building 1/4" - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex || \ - pdflatex -interaction=errorstopmode -file-line-error pmix-standard.tex < /dev/null + $(LATEX_C) -interaction=batchmode pmix-standard.tex || \ + $(LATEX_C) -interaction=errorstopmode pmix-standard.tex < /dev/null @echo "====> Building 2/4 (bibtex)" bibtex pmix-standard < /dev/null @echo "====> Building 3/4" - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex || \ - pdflatex -interaction=errorstopmode -file-line-error pmix-standard.tex < /dev/null + $(LATEX_C) -interaction=batchmode pmix-standard.tex || \ + $(LATEX_C) -interaction=errorstopmode pmix-standard.tex < /dev/null @echo "====> Building 4/4" - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex || \ - pdflatex -interaction=errorstopmode -file-line-error pmix-standard.tex < /dev/null - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex + $(LATEX_C) -interaction=batchmode pmix-standard.tex || \ + $(LATEX_C) -interaction=errorstopmode pmix-standard.tex < /dev/null + $(LATEX_C) -interaction=batchmode pmix-standard.tex @./bin/check-doc.sh @echo "====> Success" @cp pmix-standard.pdf pmix-standard-${version}.pdf @@ -93,4 +99,4 @@ check-openpmix: pmix-standard.pdf FORCECHECK @./bin/check-openpmix.py clean: - rm -f $(INTERMEDIATE_FILES) pmix-standard-*.pdf + rm -rf $(INTERMEDIATE_FILES) pmix-standard-*.pdf diff --git a/bin/process-example.py b/bin/process-example.py new file mode 100755 index 0000000..4d52705 --- /dev/null +++ b/bin/process-example.py @@ -0,0 +1,188 @@ +#!/usr/bin/python -u + +# +# Recognized tags +# : Start a block +# : End a block +# Rules: +# - The tag strings, regardless of ID, are removed from the final output +# - If multiple blocks with the same ID exist then they are concatinated together +# - ID can contain alphabetic and numberic characters and the following symbols: . _ +# - Quote marks are stripped out +# +import sys +import os +import re +import argparse +import subprocess +import shutil + +EG_BEGIN_STR="EG BEGIN ID=" +EG_END_STR="EG END ID=" +EG_ID_PATTERN="[A-Za-z0-9_\.\"]" + + +class Example: + """An example from the source code""" + eid = None + active = False + filename = None + code_block = "" + + def __init__(self): + self.eid = "" + self.active = False + self.filename = None + self.code_block = "" + + def __str__(self): + return "in_fname=[%s] id=[%s]" % (self.filename, self.eid) + + def get_out_fname(self): + f_id = self.eid.replace('"', '') + return os.path.basename(self.filename) + "_" + f_id + + def append_line(self, line): + self.code_block = self.code_block + line + if line.endswith("\n") is False: + self.code_block = self.code_block + "\n" + + def get_code_block(self): + final_block = "" + min_lead_spaces = 10000 + lines = 0 + total_lines = 0 + skip_last_lines = 0 + + # First pass to find min spacing + for line in self.code_block.splitlines(True): + if re.search(r"\s*"+EG_BEGIN_STR, line) is not None: + continue + if re.search(r"\s*"+EG_END_STR, line) is not None: + continue + + total_lines = total_lines + 1 + # Skip empty lines + if len(line) <= 1: + skip_last_lines = skip_last_lines + 1 + continue + else: + skip_last_lines = 0 + + m = re.match(r"^([ \t]+)", line) + if m is not None: + c_len = len(m.group(1)) + if c_len > 1: + min_lead_spaces = min(c_len, min_lead_spaces) + else: + # Indicates that there is a line that does not have leading spaces, but is not empty + min_lead_spaces = 0 + + # Next pass to build the string + lines = 0 + for line in self.code_block.splitlines(True): + if re.search(r"\s*"+EG_BEGIN_STR, line) is not None: + continue + if re.search(r"\s*"+EG_END_STR, line) is not None: + continue + + # Clear off trailing empty lines + if total_lines - skip_last_lines == lines: + break + lines = lines + 1 + + m = re.match(r"^([ \t]+)", line) + if m is not None: + line = line[min_lead_spaces:] + final_block = final_block + line + + return final_block + + +def process_file(filename): + """Process all of the key/value splitting in the file""" + all_examples = {} + eg_id = None + + with open(filename, 'r') as fd: + for line in fd: + line = line.rstrip() + m = re.search(r"\s*"+EG_BEGIN_STR+"("+EG_ID_PATTERN+"*)", line) + if m is not None: + eg_id = m.group(1) + # Find this object and update it + found = False + for example_id in all_examples: + if example_id == eg_id: + example = all_examples[example_id] + example.active = True + example.append_line(line) + found = True + # Insert it if not found + if found is False: + x = Example() + x.eid = eg_id + x.active = True + x.append_line(line) + x.filename = filename + all_examples[eg_id] = x + continue + + m = re.search(r"\s*"+EG_END_STR+"("+EG_ID_PATTERN+"*)", line) + if m is not None: + eg_id = m.group(1) + # Find this object and update it + for example_id in all_examples: + if example_id == eg_id: + example = all_examples[example_id] + example.active = False + example.append_line("") # Add an empty line + continue + + for example_id in all_examples: + example = all_examples[example_id] + if example.active is True: + example.append_line(line) + + return all_examples + + +if __name__ == "__main__": + # + # Command line parsing + # + parser = argparse.ArgumentParser(description="PMIx Standard Example Preprocessor") + parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") + parser.add_argument("files", nargs='+', help="List of files to process") + parser.parse_args() + args = parser.parse_args() + + # + # Create a temporary directory to store the snippets + # + gen_dir = "sources/_autogen_" + if os.access(gen_dir, os.W_OK) is False: + os.makedirs(gen_dir) + + # + # Iterate through all examples and split out the snippets + # + for f in args.files: + if os.path.exists(f) is False: + print("ERROR: File does not exist: %s" % (f)) + sys.exit(1) + + print("Processing File: %s" % (f)) + example_blocks = process_file(f) + for k, example in example_blocks.items(): + out_fname = gen_dir + "/" + example.get_out_fname() + print("\tExample: %s -- Stored in %s" % (example, out_fname)) + if args.verbose: + print("CODE BLOCK") + print("-" * 50) + print(example.get_code_block()), + print("-" * 50) + with open(out_fname, 'w') as fd: + fd.write("%s" % (example.get_code_block())) + + sys.exit(0) diff --git a/pmix.sty b/pmix.sty index d89b5be..34110aa 100644 --- a/pmix.sty +++ b/pmix.sty @@ -760,3 +760,109 @@ \newcounter{pycounter} \newcommand{\pylabel}[1]{\refstepcounter{pycounter} \label{appB:#1}} \newcommand{\refpy}[1]{\hyperref[appB:#1]{\code{#1}}} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Code examples +% Install: (if using Python 3 then use pip3) +% sudo pip2 install --upgrade pip setuptools +% sudo pip2 install Pygments +% Ubuntu: +% sudo apt update +% sudo apt install python-pip +% sudo pip3 install Pygments +% +% package to work around "No room for a new \write" message +\usepackage{morewrites} +\morewritessetup{allocate=10} +% package for code highlighting +\usepackage[chapter]{minted} +\usemintedstyle{default} +\usepackage{xcolor} + +% Background: Light Gray +\definecolor{pmixbg}{rgb}{0.95,0.95,0.95} +% Highlight: Yellow +\definecolor{pmixhl}{rgb}{0.95,0.95,0} + +%----------------------- +% C Code +% +% block of code +% \begin{pmixCodeC} +% return 0; +% \end{pmixCodeC} +% +% Inline C code +% The code \pmixCodeInlineC{printf("Hello World");} prints ``Hello World''. +% +% Import C code from a file: +% - Whole file: \pmixCodeImportC{sources/hello.c} +% - Selection of a file: \pmixCodeImportC[firstline=73, lastline=84]{sources/hello.c} +% - Highlight selection of a file: \pmixCodeImportC[firstline=73, lastline=84, highlightlines={2-4,6}]{sources/hello.c} +% +\newminted[pmixCodeC]{c}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} +\newmintinline[pmixCodeInlineC]{c}{bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + firstnumber=1} +\newmintedfile[pmixCodeImportC]{c}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} + +%----------------------- +% Python Code +% +% block of code +% \begin{pmixCodePy} +% print("Hello") +% \end{pmixCodePy} +% +% Inline C code +% The code \pmixCodeInlinePy{print("Hello World")} prints ``Hello World''. +% +% Import C code from a file: +% - Whole file: \pmixCodeImportPy{sources/hello.py} +% - Selection of a file: \pmixCodeImportPy[firstline=73, lastline=84]{sources/hello.py} +% - Highlight selection of a file: \pmixCodeImportPy[firstline=73, lastline=84, highlightlines={2-4,6}]{sources/hello.py} +% +\newminted[pmixCodepPy]{python}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} +\newmintinline[pmixCodeInlinePy]{python}{bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + firstnumber=1} +\newmintedfile[pmixCodeImportPy]{python}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/sources/README.md b/sources/README.md new file mode 100644 index 0000000..c0223dd --- /dev/null +++ b/sources/README.md @@ -0,0 +1,119 @@ +# Source Code Examples + +## General Rules + + * All examples should be buildable + * All examples should contain a comment in the header with: + - How to run the example + - Example output from a run + +## Preprocessor Syntax + +The preprocessor will automatically extract code snippets from the source files. + +The code snippets will be in the `sources/_autogen_` directory which is deleted and recreated on every build. +The code snippets will be named `sources/_autogen_/FILENAME_TAG` where `FILENAME` is the source file and `TAG` is the tag used inside the code snippet to identify the snippet. +Code snippets can span multiple sections which are concatinated, in order of appearance, into the output file. +Code snippets cannot span multiple files. + + * Start a snippet tagged `myexample` : `` + * End a snippet tagged `myexample` : `` + * Tags may only contain alphabetic and numberic characters and the following symbols: `.` `_` + - Quote marks are stripped out of the string + * Lines containing the snippet syntax as stripped out of the generated examples. + +### Example + +Given the following example program called `hello-alt.c` + +``` +#include + +int main(int argc, char **argv) +{ + // + // + pmix_status_t rc = PMIX_SUCCESS; + pmix_proc_t myproc; + // + pmix_value_t *val; + uint16_t localrank; + // + + // + rc = PMIx_Init(&myproc, NULL, 0); + if (PMIX_SUCCESS != rc) { + return 1; + } + // + + // + /* Get our rank local to this node */ + rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val); + if (PMIX_SUCCESS != rc) { + return 2; + } + localrank = val->data.uint16; + PMIX_VALUE_RELEASE(val); + // + + rc = PMIx_Finalize(NULL, 0); + return rc; +} +``` + +Produces two snippet files + +``` +Processing File: sources/hello-alt.c + Example: in_fname=[sources/hello-alt.c] id=["pmix_get"] -- Stored in sources/_autogen_/hello-alt.c_pmix_get + Example: in_fname=[sources/hello-alt.c] id=["pmix_init"] -- Stored in sources/_autogen_/hello-alt.c_pmix_init +``` + +``` +shell$ cat sources/_autogen_/hello-alt.c_pmix_init +pmix_status_t rc = PMIX_SUCCESS; +pmix_proc_t myproc; + +rc = PMIx_Init(&myproc, NULL, 0); +if (PMIX_SUCCESS != rc) { + return 1; +} +``` + +``` +shell$ cat sources/_autogen_/hello-alt.c_pmix_get +pmix_status_t rc = PMIX_SUCCESS; +pmix_proc_t myproc; +pmix_value_t *val; +uint16_t localrank; + +/* Get our rank local to this node */ +rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val); +if (PMIX_SUCCESS != rc) { + return 2; +} +localrank = val->data.uint16; +PMIX_VALUE_RELEASE(val); +``` + +## Including Code Snippets in Latex + +Import C code with and without highlighting (line numbers from generated source) +``` +Example without highlighting +\pmixCodeImportC{sources/_autogen_/hello-alt.c_pmix_init}% + +Example with highlighting: +\pmixCodeImportC[highlightlines={2-3,7,11-13}]{sources/_autogen_/hello-alt.c_pmix_get}% + +\pmixCodeInlineC{printf("Hello World");} prints ``Hello World''. +``` + +Import Python code + +``` +The inline code \pmixCodeInlinePy{print("Hello World")} prints ``Hello World''. + +\pmixCodeImportPy{sources/_autogen_/hello.py_pmix_py}% +``` \ No newline at end of file diff --git a/sources/hello.c b/sources/hello.c new file mode 100644 index 0000000..3c8cede --- /dev/null +++ b/sources/hello.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2020 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, char **argv) +{ + pid_t pid; + char hostname[1024]; + pmix_status_t rc = PMIX_SUCCESS; + pmix_proc_t myproc; + pmix_value_t *val; + uint16_t localrank; + + pid = getpid(); + gethostname(hostname, 1024); + + /* Initialize PMIx - client role */ + rc = PMIx_Init(&myproc, NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + return 1; + } + + /* Get our rank local to this node */ + rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, + "Client ns %s rank %d: PMIx_Get local rank failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + localrank = val->data.uint16; + PMIX_VALUE_RELEASE(val); + + printf("Client ns %s rank %d pid %lu: Running on host %s localrank %d\n", + myproc.nspace, myproc.rank, (unsigned long)pid, hostname , + (int)localrank); + + /* Fence to hold all processes in this namespace */ + pmix_proc_t wildproc; + PMIX_PROC_CONSTRUCT(&wildproc); + (void)strncpy(wildproc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + wildproc.rank = PMIX_RANK_WILDCARD; + + rc = PMIx_Fence(&wildproc, 1, NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, + "Client ns %s rank %d: PMIx_Fence failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + + done: + /* Finalize the PMIx library */ + printf("Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); + rc = PMIx_Finalize(NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } else { + printf("Client ns %s rank %d:PMIx_Finalize successfully completed\n", + myproc.nspace, myproc.rank); + } + + return rc; +} diff --git a/sources/hello.py b/sources/hello.py new file mode 100755 index 0000000..1b5e001 --- /dev/null +++ b/sources/hello.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +from pmix import * + +def main(): + # + client_hdl = PMIxClient() + # + print("Testing PMIx ", client_hdl.get_version()) + + print("Init") + # + info = [] + (rc,myproc) = client_hdl.init(info) + if 0 != rc: + exit(1) + # + + print("Get") + info = [] + rc, get_val = client_hdl.get(myproc, "PMIX_LOCAL_RANK", info) + print("Get result: ", rc) + print("Get value returned: ", get_val) + + print("Finalize") + info = [] + client_hdl.finalize(info) + +if __name__ == '__main__': + main()