Change warning for placeholder size to exception

docs/configuration_reference/behavior_version.rst

@@ -22,6 +22,17 @@ and not listing legacy/deprecated parameters.
 Version History
 ---------------
 
+Behavior version 15 (2022-10-19)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Dimension tags with different dynamic size tensors
+can not be merged anymore via `declare_same_as`.
+This can happen when the user did not set the dim tags
+correctly in `extern_data`.
+Otherwise it is likely a bug.
+
+See issue `#1141 <https://github.com/rwth-i6/returnn/issues/1141>`__.
+
 Behavior version 14 (2022-10-19)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

returnn/tf/util/data.py

@@ -13,7 +13,7 @@
 import traceback
 
 import returnn.util.basic as util
-from returnn.util.basic import NotSpecified, Entity
+from returnn.util.basic import BehaviorVersion, NotSpecified, Entity
 import returnn.tf.compat as tf_compat
 
 
@@ -983,6 +983,20 @@ def declare_same_as(self, other):
     if self_derived_bases.issubset(other_derived_bases):
       # Avoid cycles on derived_from_tag. https://github.com/rwth-i6/returnn/issues/1054
       return other.declare_same_as(self)
+    self._maybe_update()
+    other._maybe_update()
+    for key in set(self._same_for_batch_ctx.keys()).intersection(other._same_for_batch_ctx.keys()):
+      self_ = self._same_for_batch_ctx[key]
+      other_ = other._same_for_batch_ctx[key]
+      if not self_._validate_in_current_graph() or not other_._validate_in_current_graph():
+        continue
+      if self_.dyn_size is None or other_.dyn_size is None:
+        continue
+      BehaviorVersion.require(
+        self_.dyn_size is other_.dyn_size,
+        "%s declare_same_as %s: Invalid with different size placeholders (%r vs %r), please check external_data" % (
+          self, other, self_.dyn_size, other_.dyn_size),
+        15)
     if self_same_as is not self:
       assert not self_same_as.same_as
       if self_same_as is other_same_base:
@@ -995,15 +1009,6 @@ def declare_same_as(self, other):
     self.same_as = other_same_base
     self._same_as_tb = traceback.extract_stack()
     self._maybe_update()
-    if self.dyn_size is not None and other_same_base.dyn_size is not None:
-      if self.dyn_size is not other_same_base.dyn_size:
-        if self.batch == other_same_base.batch and self.control_flow_ctx == other_same_base.control_flow_ctx:
-          # Note: Instead of making this a warning, we could also enforce this at some point.
-          #   The user should be able to fix `extern_data` in the config such that this is correct in the first place.
-          #   Also, in addition to this warning, we might want to add some runtime check on the eq of the dyn sizes.
-          print(
-            "Warning: assuming dim tags are same with different size placeholders: %r vs %r" % (
-              self.dyn_size, other_same_base.dyn_size))
     # If we have a defined source, and this is a dynamic spatial axis, and it was undefined before,
     # maybe we can overtake the size_placeholder now.
     if other_same_base.dyn_size is not None and self.src_data:

returnn/util/basic.py

@@ -238,7 +238,7 @@ class BehaviorVersion:
   The version will be set after the config is defined at __main__.init_config() or Engine.__init__()
   """
 
-  _latest_behavior_version = 14
+  _latest_behavior_version = 15
   _behavior_version = None  # type: typing.Optional[int]
 
   @classmethod

tests/test_TFNetworkLayer.py

@@ -1178,15 +1178,13 @@ def test_CombineLayer_two_time_dims():
       name="in0", shape=(None, None, n_dim), batch_dim_axis=1, auto_create_placeholders=True)
     in1 = Data(
       # same time as first in in0
-      name="in1", shape=(None, n_dim), auto_create_placeholders=True)
+      name="in1", dim_tags=[in0.dim_tags[i] for i in (1, 0, 3)], auto_create_placeholders=True)
     in2 = Data(
       # same time as in second in in0
-      name="in2", shape=(None, n_dim), batch_dim_axis=1, auto_create_placeholders=True)
+      name="in2", dim_tags=[in0.dim_tags[i] for i in (2, 1, 3)], auto_create_placeholders=True)
     extern_data.register_data(in0)
     extern_data.register_data(in1)
     extern_data.register_data(in2)
-    in1.get_size_dim_tag(0).declare_same_as(in0.get_size_dim_tag(0))
-    in2.get_size_dim_tag(0).declare_same_as(in0.get_size_dim_tag(1))
     print("ExternData all dimension tags (allow_same_feature_dim=True):")
     pprint(extern_data.get_all_dimension_tags(allow_same_feature_dim=True))
     network = TFNetwork(config=config, extern_data=extern_data, train_flag=True)
@@ -1232,15 +1230,13 @@ def test_CombineLayer_two_time_dims_first_not_most_generic():
       name="in0", shape=(None, None, n_dim), batch_dim_axis=1, auto_create_placeholders=True)
     in1 = Data(
       # same time as first in in0
-      name="in1", shape=(None, n_dim), auto_create_placeholders=True)
+      name="in1", dim_tags=[in0.dim_tags[i] for i in (1, 0, 3)], auto_create_placeholders=True)
     in2 = Data(
       # same time as in second in in0
-      name="in2", shape=(None, n_dim), batch_dim_axis=1, auto_create_placeholders=True)
+      name="in2", dim_tags=[in0.dim_tags[i] for i in (2, 1, 3)], auto_create_placeholders=True)
     extern_data.register_data(in0)
     extern_data.register_data(in1)
     extern_data.register_data(in2)
-    in1.get_size_dim_tag(0).declare_same_as(in0.get_size_dim_tag(0))
-    in2.get_size_dim_tag(0).declare_same_as(in0.get_size_dim_tag(1))
     print("ExternData all dimension tags (allow_same_feature_dim=True):")
     pprint(extern_data.get_all_dimension_tags(allow_same_feature_dim=True))
     network = TFNetwork(config=config, extern_data=extern_data, train_flag=True)
@@ -1286,15 +1282,13 @@ def test_CombineLayer_two_time_dims_first_not_most_generic_with_n_out():
       name="in0", shape=(None, None, n_dim), batch_dim_axis=1, auto_create_placeholders=True)
     in1 = Data(
       # same time as first in in0
-      name="in1", shape=(None, n_dim), auto_create_placeholders=True)
+      name="in1", dim_tags=[in0.dim_tags[i] for i in (1, 0, 3)], auto_create_placeholders=True)
     in2 = Data(
       # same time as in second in in0
-      name="in2", shape=(None, n_dim), batch_dim_axis=1, auto_create_placeholders=True)
+      name="in2", dim_tags=[in0.dim_tags[i] for i in (2, 1, 3)], auto_create_placeholders=True)
     extern_data.register_data(in0)
     extern_data.register_data(in1)
     extern_data.register_data(in2)
-    in1.get_size_dim_tag(0).declare_same_as(in0.get_size_dim_tag(0))
-    in2.get_size_dim_tag(0).declare_same_as(in0.get_size_dim_tag(1))
     print("ExternData all dimension tags (allow_same_feature_dim=True):")
     pprint(extern_data.get_all_dimension_tags(allow_same_feature_dim=True))
     network = TFNetwork(config=config, extern_data=extern_data, train_flag=True)

tests/test_TFUtil.py

@@ -663,8 +663,8 @@ def test_Data_copy_compatible_to_bias_to_batch_time_spatial_feature():
 
 
 def test_Data_get_common_data_extra_static_spatial():
-  d1 = Data(name='t', shape=(None, 32, 128), dtype='float32', auto_create_placeholders=True)
-  d2 = Data(name='r', shape=(None, 32, 128), dtype='float32', auto_create_placeholders=True)
+  d1 = Data(name='t', shape=(None, 32, 128), dtype='float32')
+  d2 = Data(name='r', shape=(None, 32, 128), dtype='float32')
   d2.get_size_dim_tag(0).declare_same_as(d1.get_size_dim_tag(0))
   common = Data.get_common_data([d1, d2])
   assert d1.shape == common.shape
@@ -678,8 +678,8 @@ def test_Data_get_common_data_broadcast_multiple():
 
 
 def test_Data_get_common_data_extra2_static_spatial():
-  d1 = Data(name='t', shape=(None, 32, 32, 128), dtype='float32', auto_create_placeholders=True)
-  d2 = Data(name='r', shape=(None, 32, 32, 128), dtype='float32', auto_create_placeholders=True)
+  d1 = Data(name='t', shape=(None, 32, 32, 128), dtype='float32')
+  d2 = Data(name='r', shape=(None, 32, 32, 128), dtype='float32')
   d2.get_size_dim_tag(0).declare_same_as(d1.get_size_dim_tag(0))
   common = Data.get_common_data([d1, d2])
   assert d1.shape == common.shape