Semantic matching basics #4788

detox/detox.d.ts

@@ -1156,6 +1156,8 @@ declare global {
         interface NativeElement extends NativeElementActions {
         }
 
+        type SemanticMatchingTypes = 'image' | 'input-field' | 'text' | 'button' | 'scrollview' | 'list' | 'switch' | 'slider' | 'picker' | 'activity-indicator' | 'progress';
+
         interface ByFacade {
             /**
              * by.id will match an id that is given to the view via testID prop.
@@ -1185,10 +1187,19 @@ declare global {
             label(label: string | RegExp): NativeMatcher;
 
             /**
-             * Find an element by native view type.
-             * @example await element(by.type('RCTImageView'));
+             * Find an element by native view type OR semantic type.
+             * Automatically detects if the input is a semantic type or regular class name.
+             * @example 
+             * // Semantic types (cross-platform):
+             * await element(by.type('image'));
+             * await element(by.type('button'));
+             * await element(by.type('input-field'));
+             * 
+             * // Native class names (platform-specific):
+             * await element(by.type('RCTImageView'));
+             * await element(by.type('android.widget.Button'));
              */
-            type(nativeViewType: string): NativeMatcher;
+            type(typeOrSemanticType: SemanticMatchingTypes | string): NativeMatcher;
 
             /**
              * Find an element with an accessibility trait. (iOS only)

detox/ios/Detox/Invocation/Predicate.swift

@@ -147,6 +147,15 @@ class Predicate : CustomStringConvertible, CustomDebugStringConvertible {
           compoundPredicate = AndCompoundPredicate(predicates: innerPredicates, modifiers: modifiers)
         }
         return compoundPredicate
+      case Kind.or:
+        let predicatesDictionaryRepresentation = dictionaryRepresentation[Keys.predicates] as! [[String: Any]]
+        let innerPredicates = predicatesDictionaryRepresentation.compactMap { try? Predicate.with(dictionaryRepresentation: $0) }
+        guard !innerPredicates.isEmpty else {
+          fatalError("No valid predicates found in OR compound predicate")
+        }
+        return innerPredicates.count == 1
+          ? innerPredicates.first!
+          : OrCompoundPredicate(predicates: innerPredicates, modifiers: modifiers)
       default:
         fatalError("Unknown predicate type \(kind)")
     }

detox/src/android/matchers/native.js

@@ -1,9 +1,16 @@
 const DetoxRuntimeError = require('../../errors/DetoxRuntimeError');
 const invoke = require('../../invoke');
+const semanticTypes = require('../../matchers/semanticTypes');
 const { isRegExp } = require('../../utils/isRegExp');
 const { NativeMatcher } = require('../core/NativeMatcher');
 const DetoxMatcherApi = require('../espressoapi/DetoxMatcher');
 
+const createClassMatcher = (className) =>
+  new NativeMatcher(invoke.callDirectly(DetoxMatcherApi.matcherForClass(className)));
+
+const combineWithOr = (matchers) =>
+  matchers.reduce((acc, matcher) => acc.or(matcher));
+
 class LabelMatcher extends NativeMatcher {
   constructor(value) {
     super();
@@ -29,9 +36,23 @@ class IdMatcher extends NativeMatcher {
 }
 
 class TypeMatcher extends NativeMatcher {
-  constructor(value) {
+  constructor(typeOrSemanticType) {
     super();
-    this._call = invoke.callDirectly(DetoxMatcherApi.matcherForClass(value));
+
+    const descriptors = semanticTypes.getClasses(typeOrSemanticType, 'android');
+    if (!descriptors.length) {
+      throw new DetoxRuntimeError(`No class names found for: ${typeOrSemanticType}`);
+    }
+
+    const matchers = descriptors.map(({ className, excludes }) => {
+      const includeMatcher = createClassMatcher(className);
+      return excludes.length
+        ? includeMatcher.and(combineWithOr(excludes.map(createClassMatcher)).not)
+        : includeMatcher;
+    });
+
+    const combinedMatcher = combineWithOr(matchers);
+    this._call = { ...combinedMatcher._call, rawType: typeOrSemanticType };
   }
 }
 

detox/src/android/matchers/native.test.js

@@ -0,0 +1,98 @@
+// @ts-nocheck
+jest.mock('../../matchers/semanticTypes', () => ({
+  getTypes: jest.fn(),
+  getClasses: jest.fn(),
+  includes: jest.fn()
+}));
+
+
+const semanticTypes = require('../../matchers/semanticTypes');
+
+const { TypeMatcher } = require('./native');
+
+describe('Native Matchers', () => {
+  describe('TypeMatcher', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('should handle regular class names', () => {
+      semanticTypes.includes.mockReturnValue(false);
+      semanticTypes.getClasses.mockReturnValue([
+        { className: 'com.example.CustomView', excludes: [] }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('com.example.CustomView');
+      }).not.toThrow();
+    });
+
+    it('should handle semantic types automatically', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        { className: 'android.widget.ImageView', excludes: [] },
+        { className: 'com.facebook.react.views.image.ReactImageView', excludes: [] }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('image');
+      }).not.toThrow();
+    });
+
+    it('should handle exclusion objects for semantic types', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        {
+          className: 'android.widget.ProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        },
+        {
+          className: 'androidx.core.widget.ContentLoadingProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('activity-indicator');
+      }).not.toThrow();
+    });
+
+    it('should handle mixed string and exclusion objects', () => {
+      semanticTypes.includes.mockReturnValue(true);
+      semanticTypes.getClasses.mockReturnValue([
+        {
+          className: 'android.widget.ProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        },
+        {
+          className: 'androidx.core.widget.ContentLoadingProgressBar',
+          excludes: ['android.widget.AbsSeekBar']
+        }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('progress');
+      }).not.toThrow();
+    });
+
+    it('should handle regular class names when not semantic types', () => {
+      semanticTypes.includes.mockReturnValue(false);
+      semanticTypes.getClasses.mockReturnValue([
+        { className: 'android.widget.ImageView', excludes: [] }
+      ]);
+
+      expect(() => {
+        new TypeMatcher('android.widget.ImageView');
+      }).not.toThrow();
+    });
+
+    it('should throw error when no class names are found', () => {
+      semanticTypes.includes.mockReturnValue(false);
+      semanticTypes.getClasses.mockReturnValue([]);
+
+      expect(() => {
+        new TypeMatcher('empty-type');
+      }).toThrow('No class names found for: empty-type');
+    });
+  });
+});

detox/src/ios/expectTwo.js

@@ -4,7 +4,7 @@ const path = require('path');
 const fs = require('fs-extra');
 const _ = require('lodash');
 
-
+const semanticTypes = require('../matchers/semanticTypes');
 const { assertTraceDescription, assertEnum, assertNormalized } = require('../utils/assertArgument');
 const { removeMilliseconds } = require('../utils/dateUtils');
 const { actionDescription, expectDescription } = require('../utils/invocationTraceDescriptions');
@@ -17,6 +17,18 @@ const traceInvocationCall = require('../utils/traceInvocationCall').bind(null, l
 const { systemElement, systemMatcher, systemExpect, isSystemElement } = require('./system');
 const { webElement, webMatcher, webExpect, isWebElement } = require('./web');
 
+const createTypePredicate = (className) => ({ type: 'type', value: className });
+const createOrPredicate = (predicates) => ({ type: 'or', predicates });
+const createExclusionPredicate = (className, excludes) => ({
+  type: 'and',
+  predicates: [
+    createTypePredicate(className),
+    {
+      type: 'not',
+      predicate: createOrPredicate(excludes.map(createTypePredicate))
+    }
+  ]
+});
 
 const assertDirection = assertEnum(['left', 'right', 'up', 'down']);
 const assertSpeed = assertEnum(['fast', 'slow']);
@@ -440,9 +452,16 @@ class Matcher {
     return this;
   }
 
-  type(type) {
-    if (typeof type !== 'string') throw new Error('type should be a string, but got ' + (type + (' (' + (typeof type + ')'))));
-    this.predicate = { type: 'type', value: type };
+    type(typeOrSemanticType) {
+    if (typeof typeOrSemanticType !== 'string') throw new Error('type should be a string, but got ' + (typeOrSemanticType + (' (' + (typeof typeOrSemanticType + ')'))));
+
+    const descriptors = semanticTypes.getClasses(typeOrSemanticType, 'ios');
+    const predicates = descriptors.map(({ className, excludes }) =>
+      excludes.length ? createExclusionPredicate(className, excludes) : createTypePredicate(className)
+    );
+
+    this.predicate = predicates.length > 1 ? createOrPredicate(predicates) : predicates[0];
+    this.predicate.rawType = typeOrSemanticType;
     return this;
   }
 

detox/src/ios/expectTwo.test.js

@@ -793,6 +793,89 @@ describe('expectTwo', () => {
     });
   });
 
+  describe('semantic types', () => {
+    it(`should parse correct JSON for regular class name using by.type()`, async () => {
+      const testCall = await e.element(e.by.type('CustomUIView')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'type',
+            value: 'CustomUIView',
+            rawType: 'CustomUIView'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+
+    it(`should parse correct JSON for semantic type 'image' using by.type()`, async () => {
+      const testCall = await e.element(e.by.type('image')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'or',
+            predicates: [
+              {
+                type: 'type',
+                value: 'RCTImageView'
+              },
+              {
+                type: 'type',
+                value: 'RCTImageComponentView'
+              },
+              {
+                type: 'type',
+                value: 'UIImageView'
+              },
+            ],
+            rawType: 'image'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+
+    it(`should parse correct JSON for semantic type with exclusions using by.type()`, async () => {
+      const semanticTypes = require('../matchers/semanticTypes');
+      const originalGetClasses = semanticTypes.getClasses;
+
+      semanticTypes.getClasses = jest.fn().mockReturnValue([
+        { className: 'UIActivityIndicatorView', excludes: ['UIProgressView'] }
+      ]);
+
+      const testCall = await e.element(e.by.type('mock-type-with-exclusion')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'and',
+            predicates: [
+              { type: 'type', value: 'UIActivityIndicatorView' },
+              {
+                type: 'not',
+                predicate: {
+                  type: 'or',
+                  predicates: [{ type: 'type', value: 'UIProgressView' }]
+                }
+              }
+            ],
+            rawType: 'mock-type-with-exclusion'
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+      semanticTypes.getClasses = originalGetClasses;
+    });
+  });
+
   describe('web views', () => {
     it(`should parse expect(web(by.id('webViewId').element(web(by.label('tapMe')))).toExist()`, async () => {
       const testCall = await e.expect(e.web(e.by.id('webViewId')).atIndex(1).element(e.by.web.label('tapMe')).atIndex(2)).toExist();