Update unit tests to demonstrate #148

This can be observed by running `testimony -n --config tests/config-full.yaml validate tests`

test_case_mismatch_case_insensitive_values will be flagged for the valid, case correct tokens CaseImportance and CaseAutomation

testimony/config.py

@@ -51,7 +51,7 @@ class TokenConfig(object):
     """
     Represent config for one token.
 
-    Curently only checks for value.
+    Currently only checks for value.
     """
 
     def __init__(self, name, config):

tests/config-full.yaml

@@ -19,12 +19,15 @@ steps: {}
 tags: {}
 type: {}
 caseautomation:
+    # case sensitive by default
     type: choice
     choices:
         - Automated
         - NotAutomated
         - ManualOnly
 caseimportance:
+    # case sensitive explicitly
+    casesensitive: True
     type: choice
     choices:
         - Critical

tests/sample_output.txt

@@ -183,7 +183,7 @@ RST parser messages:
 
 
 
-ConfigurationFileTestCase::test_lowercase_key:171
+ConfigurationFileTestCase::test_lowercase_key:177
 -------------------------------------------------
 
 Assert:
@@ -202,7 +202,7 @@ Test:
  Check that key can be provided in all lowercase.
 
 
-ConfigurationFileTestCase::test_mixedcase_key:182
+ConfigurationFileTestCase::test_mixedcase_key:188
 -------------------------------------------------
 
 Assert:
@@ -218,7 +218,7 @@ Test:
  Check that key can be provided in mixed case.
 
 
-ConfigurationFileTestCase::test_overwrite_key_with_variable_case:191
+ConfigurationFileTestCase::test_overwrite_key_with_variable_case:197
 --------------------------------------------------------------------
 
 Assert:
@@ -234,7 +234,7 @@ Test:
  Check overwriting keys that use different casing on various levels.
 
 
-ConfigurationFileTestCase::test_multiple_invalid_keys:202
+ConfigurationFileTestCase::test_multiple_invalid_keys:208
 ---------------------------------------------------------
 
 Assert:
@@ -255,7 +255,7 @@ Test:
 Unexpected tokens:
   Caseimportance: Lowest
 
-ConfigurationFileTestCase::test_case_mismatch_case_insensitive_values:215
+ConfigurationFileTestCase::test_case_mismatch_case_insensitive_values:221
 -------------------------------------------------------------------------
 
 Assert:
@@ -273,8 +273,11 @@ Status:
 Test:
  Check 'Status' value is case insensitive.
 
+Unexpected tokens:
+  Caseautomation: Automated
+  Caseimportance: Critical
 
-ConfigurationFileTestCase::test_case_mismatch_case_sensitive_values:226
+ConfigurationFileTestCase::test_case_mismatch_case_sensitive_values:236
 -----------------------------------------------------------------------
 
 Assert:
@@ -290,12 +293,13 @@ Status:
  manual
 
 Test:
- Check 'CaseAutomation' value is case sensitive.
+ Check 'CaseAutomation' and 'CaseImportance' validation is case sensitive.
 
 Unexpected tokens:
   Caseautomation: AUTOMATED
+  Caseimportance: critical
 
-DecoratorsTestCase::test_no_decorator:247
+DecoratorsTestCase::test_no_decorator:260
 -----------------------------------------
 
 Assert:
@@ -311,7 +315,7 @@ Test:
  Test without decorator.
 
 
-DecoratorsTestCase::test_one_decorator:251
+DecoratorsTestCase::test_one_decorator:264
 ------------------------------------------
 
 Assert:
@@ -330,7 +334,7 @@ Test:
  Test with one decorator.
 
 
-DecoratorsTestCase::test_multiple_decorators:256
+DecoratorsTestCase::test_multiple_decorators:269
 ------------------------------------------------
 
 Assert:
@@ -349,7 +353,7 @@ Test:
  Test with multiple decorators.
 
 
-MergeDecoratorsTestCase::test_no_decorator:271
+MergeDecoratorsTestCase::test_no_decorator:285
 ----------------------------------------------
 
 Assert:
@@ -368,7 +372,7 @@ Test:
  Test without decorator.
 
 
-MergeDecoratorsTestCase::test_decorator:275
+MergeDecoratorsTestCase::test_decorator:289
 -------------------------------------------
 
 Assert:
@@ -472,23 +476,31 @@ RSTFormattingTestCase::test_invalid_list_style:150
 
 
 
-ConfigurationFileTestCase::test_multiple_invalid_keys:202
+ConfigurationFileTestCase::test_multiple_invalid_keys:208
 ---------------------------------------------------------
 
 * Unexpected tokens:
   Caseimportance: Lowest
 
-ConfigurationFileTestCase::test_case_mismatch_case_sensitive_values:226
+ConfigurationFileTestCase::test_case_mismatch_case_insensitive_values:221
+-------------------------------------------------------------------------
+
+* Unexpected tokens:
+  Caseautomation: Automated
+  Caseimportance: Critical
+
+ConfigurationFileTestCase::test_case_mismatch_case_sensitive_values:236
 -----------------------------------------------------------------------
 
 * Unexpected tokens:
   Caseautomation: AUTOMATED
+  Caseimportance: critical
 
 Total number of tests: 21
-Total number of invalid docstrings: 6 (28.57%)
+Total number of invalid docstrings: 7 (33.33%)
 Test cases with no docstrings: 1 (4.76%)
 Test cases missing minimal docstrings: 3 (14.29%)
-Test cases with unexpected tags: 3 (14.29%)
+Test cases with unexpected tags: 4 (19.05%)
 Test cases with unexpected token values in docstrings: 0 (0.00%)
 Test cases with unparseable docstrings: 1 (4.76%)
 
@@ -534,18 +546,19 @@ RSTFormattingTestCase::test_invalid_list_style:150
 
 
 
-ConfigurationFileTestCase::test_multiple_invalid_keys:202
+ConfigurationFileTestCase::test_multiple_invalid_keys:208
 ---------------------------------------------------------
 
 * Tokens with invalid values:
   Caseimportance: Lowest (type: 'choice')
   Status: Invalid (type: 'choice')
 
-ConfigurationFileTestCase::test_case_mismatch_case_sensitive_values:226
+ConfigurationFileTestCase::test_case_mismatch_case_sensitive_values:236
 -----------------------------------------------------------------------
 
 * Tokens with invalid values:
   Caseautomation: AUTOMATED (type: 'choice')
+  Caseimportance: critical (type: 'choice')
 
 Total number of tests: 21
 Total number of invalid docstrings: 6 (28.57%)

tests/test_sample.py

@@ -166,7 +166,13 @@ def test_invalid_list_style(self):
 
 
 class ConfigurationFileTestCase():
-    """Class to test Testimony config file support."""
+    """Class to test Testimony config file support.
+
+    Behavior was observed in GitHub issue #148 where case sensitivity was not applied correctly
+    The failure was missed by the original unit test test_case_mismatch_case_sensitive_values
+    Because it was the only test using the case sensitive token.
+    CaseImportance and CaseAutomation tokens are now included in cases not directly testing them
+    """
 
     def test_lowercase_key(self):
         """Check that key can be provided in all lowercase.
@@ -220,22 +226,29 @@ def test_case_mismatch_case_insensitive_values(self):
         :Assert: Value doesn't have to match case
 
         :Status: MANUAL
+
+        :CaseImportance: Critical
+
+        :CaseAutomation: Automated
         """
         pass
 
     def test_case_mismatch_case_sensitive_values(self):
-        """Check 'CaseAutomation' value is case sensitive.
+        """Check 'CaseAutomation' and 'CaseImportance' validation is case sensitive.
 
         :Feature: Config file support
 
         :Assert: Value has to match case
 
         :Status: manual
 
+        :CaseImportance: critical
+
         :CaseAutomation: AUTOMATED
         """
         pass
 
+
 class DecoratorsTestCase():
     """Class to test Testimony support for decorators
 
@@ -259,6 +272,7 @@ def test_multiple_decorators(self):
         """Test with multiple decorators."""
         pass
 
+
 @tier1
 class MergeDecoratorsTestCase():
     """Class to test Testimony support for decorated classes