From 945a39731f1b8b6b1a9394af6c4acc687afa324e Mon Sep 17 00:00:00 2001
From: Peter Ohler <peter@ohler.com>
Date: Mon, 14 Aug 2023 20:15:40 -0400
Subject: [PATCH] Try out different max_nesting check

---
 ext/oj/dump_compat.c                 | 13 +++++++++++++
 notes                                |  6 ++++++
 test/json_gem/json_generator_test.rb |  2 +-
 test/test_compat.rb                  | 13 +++++++++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/ext/oj/dump_compat.c b/ext/oj/dump_compat.c
index d09f46e6..ea3f11c6 100644
--- a/ext/oj/dump_compat.c
+++ b/ext/oj/dump_compat.c
@@ -870,7 +870,12 @@ void oj_dump_compat_val(VALUE obj, int depth, Out out, bool as_ok) {
     int type = rb_type(obj);
 
     TRACE(out->opts->trace, "dump", obj, depth, TraceIn);
+		// The max_nesting logic is that an empty Array or Hash is assumed to have
+		// content so the max_nesting should fail but a non-collection value is
+		// okay. That means a check for a collectable value is needed before
+		// raising.
     if (out->opts->dump_opts.max_depth <= depth) {
+#if 0
         // When JSON.dump is called then an ArgumentError is expected and the
         // limit is the depth inclusive. If JSON.generate is called then a
         // NestingError is expected and the limit is inclusive. Worse than
@@ -886,6 +891,14 @@ void oj_dump_compat_val(VALUE obj, int depth, Out out, bool as_ok) {
             }
             raise_json_err("Too deeply nested", "NestingError");
         }
+#else
+				if (RUBY_T_ARRAY == type || RUBY_T_HASH == type) {
+						if (0 < out->argc) {
+								set_state_depth(*out->argv, depth);
+						}
+						raise_json_err("Too deeply nested", "NestingError");
+				}
+#endif
     }
     if (0 < type && type <= RUBY_T_FIXNUM) {
         DumpFunc f = compat_funcs[type];
diff --git a/notes b/notes
index 832b280b..5c33b73f 100644
--- a/notes
+++ b/notes
@@ -6,6 +6,12 @@
 - skip_null_byte => omit_null_byte
  - and move to dump parameters
 
+- max_nesting
+ - JSON.dump([[]], n) - n = 2 ok, n = 1 fails
+ - generate is the same
+ - fast_generate is the same
+ - pretty_generate is the same
+ - [[]].to_json same
 
 
 - stream writes
diff --git a/test/json_gem/json_generator_test.rb b/test/json_gem/json_generator_test.rb
index 5c9c8de1..fe5d590b 100755
--- a/test/json_gem/json_generator_test.rb
+++ b/test/json_gem/json_generator_test.rb
@@ -352,7 +352,7 @@ def test_nesting
     too_deep_ary = eval too_deep
     assert_raise(JSON::NestingError) { JSON.generate too_deep_ary }
     assert_raise(JSON::NestingError) { JSON.generate too_deep_ary, :max_nesting => 100 }
-    ok = JSON.generate too_deep_ary, :max_nesting => 101
+    ok = JSON.generate too_deep_ary, :max_nesting => 102
     assert_equal too_deep, ok
     ok = JSON.generate too_deep_ary, :max_nesting => nil
     assert_equal too_deep, ok
diff --git a/test/test_compat.rb b/test/test_compat.rb
index d41e055e..e1ef5dca 100755
--- a/test/test_compat.rb
+++ b/test/test_compat.rb
@@ -474,6 +474,19 @@ def test_arg_passing
   def test_max_nesting
     assert_raises() { Oj.to_json([[[[[]]]]], :max_nesting => 3) }
     assert_raises() { Oj.dump([[[[[]]]]], :max_nesting => 3, :mode=>:compat) }
+
+    assert_raises() { Oj.to_json([[]], :max_nesting => 1) }
+    assert_equal('[[]]', Oj.to_json([[]], :max_nesting => 2))
+
+    assert_raises() { Oj.dump([[]], :max_nesting => 1, :mode=>:compat) }
+    assert_equal('[[]]', Oj.dump([[]], :max_nesting => 2, :mode=>:compat))
+
+    assert_raises() { Oj.to_json([[3]], :max_nesting => 1) }
+    assert_equal('[[3]]', Oj.to_json([[3]], :max_nesting => 2))
+
+    assert_raises() { Oj.dump([[3]], :max_nesting => 1, :mode=>:compat) }
+    assert_equal('[[3]]', Oj.dump([[3]], :max_nesting => 2, :mode=>:compat))
+
   end
 
   def test_bad_unicode