build: fixing unneeded rebuild problem and resolving buildbot issues

Author: Sam Kleinman

bin/mongodb_docs_meta.py bin/archive/mongodb_docs_meta.pybin/mongodb_docs_meta.py renamed to bin/archive/mongodb_docs_meta.py

@@ -1,4 +1,4 @@
-#!/usr/bin/pythono
+#!/usr/bin/python
 """
 This module defines a class.
 """
@@ -8,21 +8,10 @@
 
 MANUAL_BRANCH = "manual"
 
-def shell_value( args ):
-    import subprocess
-    if isinstance( args , str ):
-        r = re.compile( "\s+" )
-        args = r.split( args )
-    p = subprocess.Popen( args , stdout=subprocess.PIPE , stderr=subprocess.PIPE )
-    r = p.communicate()
-    value = r[0].decode().rstrip()
-    return value
-
 class VersionMeta():
     def __init__(self):
         self.branch = shell_value('git symbolic-ref HEAD').split('/')[2]
         self.commit = shell_value('git rev-parse --verify HEAD')
-        self.current_year = str(datetime.date.today().year)
 
         if self.branch == MANUAL_BRANCH:
             self.manual_path = "manaul"
@@ -35,8 +24,7 @@ def main():
 
     print("MongoDB Manual:" + BREAK +
           "     Commit: " + meta.commit + BREAK +
-          "     Branch: " + meta.branch + BREAK +
-          "     Current year: " + meta.current_year)
+          "     Branch: " + meta.branch + BREAK)
 
 if __name__ == "__main__":
     main()

bin/update_hash.py

@@ -1,9 +1,14 @@
 #!/usr/bin/python
 
-import mongodb_docs_meta
+import os 
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../")))
+
+from conf import VersionMeta
 
 def main():
-    meta = mongodb_docs_meta.VersionMeta()
+    meta = VersionMeta()
 
     content = '.. |commit| replace:: ``' + meta.commit + '``' + '\n'
 

conf.py

@@ -8,9 +8,34 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../bin")))
-from mongodb_docs_meta import VersionMeta
+import sys
+import os
+import datetime
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "bin")))
+
+MANUAL_BRANCH = 'manual'
+
+def shell_value( args ):
+    import re
+    import subprocess
+    if isinstance( args , str ):
+        r = re.compile( "\s+" )
+        args = r.split( args )
+    p = subprocess.Popen( args , stdout=subprocess.PIPE , stderr=subprocess.PIPE )
+    r = p.communicate()
+    value = r[0].decode().rstrip()
+    return value
+
+class VersionMeta():
+    def __init__(self):
+        self.branch = shell_value('git symbolic-ref HEAD').split('/')[2]
+        self.commit = shell_value('git rev-parse --verify HEAD')
+
+        if self.branch == MANUAL_BRANCH:
+            self.manual_path = MANUAL_BRANCH
+        else:
+            self.manual_path = self.branch
 
 meta = VersionMeta()
 
@@ -23,7 +48,7 @@
 extensions = ["sphinx.ext.intersphinx", "sphinx.ext.extlinks", "sphinx.ext.todo", "mongodb_domain", "additional_directives", "aggregation_domain"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['../../.templates']
+templates_path = ['.templates']
 
 # The suffix of source filenames.
 source_suffix = '.txt'
@@ -36,7 +61,7 @@
 
 # General information about the project.
 project = u'MongoDB'
-copyright = u'2011-' + meta.current_year + ', 10gen, Inc.'
+copyright = u'2011-' + str(datetime.date.today().year) + ', 10gen, Inc.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -113,7 +138,7 @@
 # html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = ['../../themes']
+html_theme_path = ['themes']
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
@@ -124,7 +149,7 @@
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-html_logo = "source/.static/logo-mongodb.png"
+html_logo = ".static/logo-mongodb.png"
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
@@ -134,7 +159,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['../../source/.static']
+html_static_path = ['source/.static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
@@ -222,7 +247,7 @@
 epub_title = u'MongoDB'
 epub_author = u'MongoDB Documentation Project'
 epub_publisher = u'MongoDB Documentation Project'
-epub_copyright = u'2011-' + meta.current_year + ', 10gen Inc.'
+epub_copyright = u'2011-' + str(datetime.date.today().year) + ', 10gen Inc.'
 epub_theme = 'epub_mongodb'
 epub_tocdup = True
 epub_tocdepth = 3

makefile

@@ -26,9 +26,9 @@ else
 current-if-not-manual = $(current-branch)
 endif
 
-sphinx-conf = $(branch-output)/conf.py
 # Sphinx variables.
-SPHINXOPTS = -c $(branch-output)
+sphinx-conf = $(branch-output)/conf.py
+SPHINXOPTS = -c ./
 SPHINXBUILD = sphinx-build
 
 ifdef NITPICK
@@ -112,14 +112,14 @@ endif
 # access these targets through the ``publish`` target.
 .PHONY: initial-dependencies static-components sphinx-components post-processing
 
-pre-build-dependencies:source/includes/hash.rst source/about.txt
+pre-build-dependencies:setup source/includes/hash.rst source/about.txt
 initial-dependencies:$(public-branch-output)/MongoDB-Manual.epub
 	@echo [build]: completed the pre-publication routine for the $(manual-branch) branch of the Manual.
 static-components:$(public-output)/index.html $(public-output)/10gen-gpg-key.asc $(public-branch-output)/.htaccess $(public-branch-output)/release.txt $(public-output)/osd.xml
 	@echo [build]: completed building and migrating all non-Sphinx components of the build.
 post-processing:error-pages links
 	@echo [build]: completed all post processing steps.
-sphinx-components:$(public-branch-output)/ $(public-branch-output)/sitemap.xml.gz $(public-branch-output)/MongoDB-Manual.pdf $(public-branch-output)/single $(public-branch-output)/single/index.html
+sphinx-components:$(public-branch-output)/MongoDB-Manual.pdf $(public-branch-output)/single $(public-branch-output)/single/index.html $(public-branch-output)/ $(public-branch-output)/sitemap.xml.gz 
 	@echo [build]: completed the publication routine for all Sphinx Components of the Manual Build.
 
 #
@@ -133,7 +133,7 @@ setup:source/includes/hash.rst
 	@echo [build]: created $(public-branch-output)
 
 source/includes/hash.rst:
-	@./bin/update_hash.py
+	@$(PYTHONBIN) bin/update_hash.py
 	@echo [build]: \(re\)generated $@.
 source/about.txt:setup
 	@touch $@
@@ -155,7 +155,7 @@ $(branch-output)/singlehtml/contents.html:$(branch-output)/singlehtml
 $(branch-output)/latex/MongoDB.tex:latex
 $(branch-output)/latex/MongoDB.pdf:$(branch-output)/latex/MongoDB-Manual.tex
 $(branch-output)/latex/MongoDB-Manual.tex:$(branch-output)/latex/MongoDB.tex
-	@python bin/copy-if-needed.py -i $< -o $@ -b pdf
+	@$(PYTHONBIN) bin/copy-if-needed.py -i $< -o $@ -b pdf
 $(public-branch-output)/MongoDB-Manual-$(current-branch).pdf:$(branch-output)/latex/MongoDB-Manual.pdf
 	@cp $< $@
 	@echo [build]: migrated $@
@@ -253,8 +253,26 @@ clean-all:
 	-rm -rf $(output)/*
 
 # Needed for all sphinx builds.
-$(sphinx-conf):
-	@python bin/copy-if-needed.py -i conf.py -o $@ -b sphinx
+.PHONY: $(branch-output)/themes $(branch-output)/bin $(branch-output)/.static $(branch-output)/.templates
+
+# $(sphinx-conf):
+# 	@$(PYTHONBIN) bin/copy-if-needed.py -i conf.py -o $@ -b sphinx
+# $(branch-output)/themes:themes
+# 	@mkdir -p $@
+# 	@rsync --recursive --delete $</  $@
+# 	@echo [build]: syncing '$<' directory
+# $(branch-output)/bin:bin
+# 	@mkdir -p $@
+# 	@rsync  --recursive --delete $</  $@
+# 	@echo [build]: syncing '$<' directory
+# $(branch-output)/.templates:.templates
+# 	@mkdir -p $@
+# 	@rsync --recursive --delete $</ $@
+# 	@echo [build]: syncing '$<' directory
+# $(branch-output)/.static:source/.static
+# 	@mkdir -p $@
+# 	@rsync --recursive --delete $</ $@
+# 	@echo [build]: syncing '$<' directory
 
 ######################################################################
 #
@@ -263,19 +281,19 @@ $(sphinx-conf):
 ######################################################################
 
 .PHONY: html dirhtml singlehtml epub sitemap
-html:$(sphinx-conf)
+html:
 	@echo [html]: build starting at `date`.
 	@mkdir -p $(branch-output)/html
 	@echo [html]: created $(branch-output)/html
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(branch-output)/html
 	@echo [html]: build complete at `date`.
-dirhtml:$(sphinx-conf)
+dirhtml:
 	@echo [dirhtml]: build starting at `date`.
 	@mkdir -p $(branch-output)/dirhtml
 	@echo [dirhtml]: created $(branch-output)/dirhtml
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(branch-output)/dirhtml
 	@echo [dirhtml]: build complete at `date`.
-singlehtml:$(sphinx-conf)
+singlehtml:
 	@echo [singlehtml]: build started at `date`.
 	@mkdir -p $(branch-output)/singlehtml
 	@echo [singlehtml]: created $(branch-output)/singlehtml
@@ -284,7 +302,7 @@ singlehtml:$(sphinx-conf)
 
 epub-command = $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(branch-output)/epub
 epub-filter = sed $(SED_ARGS_REGEX) -e '/^WARNING: unknown mimetype.*ignoring$$/d' -e '/^WARNING: search index.*incomplete.$$/d'
-epub:pre-build-dependencies $(sphinx-conf)
+epub: pre-build-dependencies
 	@echo [epub]: starting epub build at `date`.
 	@mkdir -p $(branch-output)/epub
 	@echo [epub]: created $(branch-output)/epub
@@ -320,7 +338,7 @@ $(branch-output)/sitemap.xml.gz:$(public-output)/manual
 UNCOMPRESSED_MAN := $(wildcard $(branch-output)/man/*.1)
 COMPRESSED_MAN := $(subst .1,.1.gz,$(UNCOMPRESSED_MAN))
 
-man:$(sphinx-conf)
+man:
 	@echo [man]: starting man build at `date`.
 	@mkdir -p $(branch-output)/man
 	@echo [build]: created $(branch-output)/man
@@ -342,13 +360,13 @@ $(branch-output)/man/%.1.gz: $(branch-output)/man/%.1
 .PHONY: aspirational aspiration draft draft-pdf draft-pdfs
 aspiration:draft
 aspirational:draft
-draft:$(sphinx-conf)
+draft:
 	@echo [draft]: draft-html started at `date`.
 	@mkdir -p $(branch-output)/draft
 	@echo [draft]: created $(branch-output)/draft
 	$(SPHINXBUILD) -b html $(DRAFTSPHINXOPTS) $(branch-output)/draft
 	@echo [draft]: draft-html build finished at `date`.
-draft-latex:$(sphinx-conf)
+draft-latex:
 	@echo [draft]: draft-latex build started at `date`.
 	@mkdir -p $(branch-output)/draft-latex
 	@echo [draft]: created $(branch-output)/draft-latex
@@ -366,25 +384,25 @@ draft-pdfs:draft-latex draft-pdf
 ##########################################################################
 
 .PHONY: changes linkcheck json doctest
-json:$(sphinx-conf)
+json:
 	@echo [json]: build started at `date`.
 	@mkdir -p $(branch-output)/json
 	@echo [json]: created $(branch-output)/json
 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(branch-output)/json
 	@echo [json]: build finished at `date`.
-changes:$(sphinx-conf)
+changes:
 	@echo [changes]: build started at `date`.
 	@mkdir -p $(branch-output)/changes
 	@echo [changes]: created $(branch-output)/changes
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(branch-output)/changes
 	@echo [changes]: build finished at `date`.
-linkcheck:$(sphinx-conf)
+linkcheck:
 	@echo [link]: build started at `date`.
 	@mkdir -p $(branch-output)/linkcheck
 	@echo [link]: created $(branch-output)/linkcheck
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(branch-output)/linkcheck
 	@echo [link]: Link check complete at `date`. See $(branch-output)/linkcheck/output.txt.
-doctest:$(sphinx-conf)
+doctest:
 	@echo [test]: build started at `date`.
 	@mkdir -p $(branch-output)/doctest
 	@echo [test]: created $(branch-output)/doctest
@@ -399,7 +417,7 @@ doctest:$(sphinx-conf)
 
 .PHONY:pdfs latex latexpdf
 
-latex:$(sphinx-conf) $(hash-output-file)
+latex:
 	@echo [latex]: starting TeX file generation at `date`.
 	@mkdir -p $(branch-output)/latex
 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(branch-output)/latex