Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix scroll action in chrome extension #194

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+176 −77
Open

fix: fix scroll action in chrome extension #194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/site/docs/zh/index.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Midscene.js 采用了多模态大语言模型（LLM），能够直观地“理

```typescript
// 👀 输入关键字，执行搜索
// 注：尽管这是一个英文页面，你也可以用中文指令控制它
// 尽管这是一个英文页面，你也可以用中文指令控制它
await ai('在搜索框输入 "Headphones" ，敲回车');

// 👀 找到列表里耳机相关的信息
Expand Down
62 changes: 41 additions & 21 deletions packages/web-integration/src/appium/page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import fs from 'node:fs';
import type { Size } from '@midscene/core/.';
import type { Point, Size } from '@midscene/core/.';
import { getTmpFile } from '@midscene/core/utils';
import { base64Encoded, resizeImg } from '@midscene/shared/img';
import { DOMParser } from '@xmldom/xmldom';
Expand Down Expand Up @@ -104,62 +104,82 @@ export class Page implements AbstractPage {
}

// Scroll to top element
async scrollUntilTop(distance?: number): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;
async scrollUntilTop(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, -scrollDistance, 100);
await this.mouseWheel(0, -9999999, 100);
}

// Scroll to bottom element
async scrollUntilBottom(distance?: number): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;
async scrollUntilBottom(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, scrollDistance, 100);
await this.mouseWheel(0, 9999999, 100);
}

async scrollUntilLeft(distance?: number): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;
async scrollUntilLeft(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(-scrollDistance, 0, 100);
await this.mouseWheel(-9999999, 0, 100);
}

async scrollUntilRight(distance?: number): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;
async scrollUntilRight(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(scrollDistance, 0, 100);
await this.mouseWheel(9999999, 0, 100);
}

// Scroll up one screen
async scrollUp(distance?: number): Promise<void> {
async scrollUp(distance?: number, startingPoint?: Point): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, -scrollDistance, 1000);
}

// Scroll down one screen
async scrollDown(distance?: number): Promise<void> {
async scrollDown(distance?: number, startingPoint?: Point): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, scrollDistance, 1000);
}

async scrollLeft(distance?: number): Promise<void> {
async scrollLeft(distance?: number, startingPoint?: Point): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(-scrollDistance, 0, 1000);
}

async scrollRight(distance?: number): Promise<void> {
async scrollRight(distance?: number, startingPoint?: Point): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(scrollDistance, 0, 1000);
}

Expand Down
69 changes: 53 additions & 16 deletions packages/web-integration/src/chrome-extension/page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import fs from 'node:fs';
import type { WebKeyInput } from '@/common/page';
import type { ElementInfo } from '@/extractor';
import type { AbstractPage } from '@/page';
import type { Rect, Size } from '@midscene/core/.';
import type { Point, Rect, Size } from '@midscene/core/.';
import { ifInBrowser } from '@midscene/shared/utils';
import type { Protocol as CDPTypes } from 'devtools-protocol';

Expand Down Expand Up @@ -184,44 +184,76 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
return url || '';
}

async scrollUntilTop() {
async scrollUntilTop(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(0, -9999999);
}

async scrollUntilBottom() {
async scrollUntilBottom(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(0, 9999999);
}

async scrollUntilLeft() {
async scrollUntilLeft(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(-9999999, 0);
}

async scrollUntilRight() {
async scrollUntilRight(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(9999999, 0);
}

async scrollUp(distance?: number) {
async scrollUp(distance?: number, startingPoint?: Point) {
const { height } = await this.size();
const scrollDistance = distance || height * 0.7;
return this.mouse.wheel(0, -scrollDistance);
return this.mouse.wheel(
0,
-scrollDistance,
startingPoint?.left,
startingPoint?.top,
);
}

async scrollDown(distance?: number) {
async scrollDown(distance?: number, startingPoint?: Point) {
const { height } = await this.size();
const scrollDistance = distance || height * 0.7;
return this.mouse.wheel(0, scrollDistance);
return this.mouse.wheel(
0,
scrollDistance,
startingPoint?.left,
startingPoint?.top,
);
}

async scrollLeft(distance?: number) {
async scrollLeft(distance?: number, startingPoint?: Point) {
const { width } = await this.size();
const scrollDistance = distance || width * 0.7;
return this.mouse.wheel(-scrollDistance, 0);
return this.mouse.wheel(
-scrollDistance,
0,
startingPoint?.left,
startingPoint?.top,
);
}

async scrollRight(distance?: number) {
async scrollRight(distance?: number, startingPoint?: Point) {
const { width } = await this.size();
const scrollDistance = distance || width * 0.7;
return this.mouse.wheel(scrollDistance, 0);
return this.mouse.wheel(
scrollDistance,
0,
startingPoint?.left,
startingPoint?.top,
);
}

async clearInput(element: ElementInfo) {
Expand Down Expand Up @@ -267,11 +299,16 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
clickCount: 1,
});
},
wheel: async (deltaX: number, deltaY: number) => {
wheel: async (
deltaX: number,
deltaY: number,
startX?: number,
startY?: number,
) => {
await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
type: 'mouseWheel',
x: 10,
y: 10,
x: startX || 10,
y: startY || 10,
deltaX,
deltaY,
});
Expand Down
42 changes: 28 additions & 14 deletions packages/web-integration/src/common/tasks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,35 +314,49 @@ export class PageTaskExecutor {
thought: plan.thought,
locate: plan.locate,
executor: async (taskParam, { element }) => {
if (element) {
await this.page.mouse.move(
element.center[0],
element.center[1],
);
}
const startingPoint = element
? {
left: element.center[0],
top: element.center[1],
}
: undefined;
const scrollToEventName = taskParam?.scrollType;
if (scrollToEventName === 'untilTop') {
await this.page.scrollUntilTop();
await this.page.scrollUntilTop(startingPoint);
} else if (scrollToEventName === 'untilBottom') {
await this.page.scrollUntilBottom();
await this.page.scrollUntilBottom(startingPoint);
} else if (scrollToEventName === 'untilRight') {
await this.page.scrollUntilRight();
await this.page.scrollUntilRight(startingPoint);
} else if (scrollToEventName === 'untilLeft') {
await this.page.scrollUntilLeft();
await this.page.scrollUntilLeft(startingPoint);
} else if (scrollToEventName === 'once') {
if (taskParam.direction === 'down') {
await this.page.scrollDown(taskParam.distance || undefined);
await this.page.scrollDown(
taskParam.distance || undefined,
startingPoint,
);
} else if (taskParam.direction === 'up') {
await this.page.scrollUp(taskParam.distance || undefined);
await this.page.scrollUp(
taskParam.distance || undefined,
startingPoint,
);
} else if (taskParam.direction === 'left') {
await this.page.scrollLeft(taskParam.distance || undefined);
await this.page.scrollLeft(
taskParam.distance || undefined,
startingPoint,
);
} else if (taskParam.direction === 'right') {
await this.page.scrollRight(taskParam.distance || undefined);
await this.page.scrollRight(
taskParam.distance || undefined,
startingPoint,
);
} else {
throw new Error(
`Unknown scroll direction: ${taskParam.direction}`,
);
}
// until mouse event is done
await sleep(500);
} else {
throw new Error(
`Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
Expand Down
16 changes: 8 additions & 8 deletions packages/web-integration/src/page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { Size } from '@midscene/core/.';
import type { Point, Size } from '@midscene/core/.';
import type { WebKeyInput } from './common/page';
import type { WebUIContext } from './common/utils';
import type { ElementInfo } from './extractor';
Expand Down Expand Up @@ -33,13 +33,13 @@ export abstract class AbstractPage {

async clearInput(element: ElementInfo): Promise<void> {}

abstract scrollUntilTop(): Promise<void>;
abstract scrollUntilBottom(): Promise<void>;
abstract scrollUntilLeft(): Promise<void>;
abstract scrollUntilRight(): Promise<void>;
abstract scrollUp(distance?: number): Promise<void>;
abstract scrollDown(distance?: number): Promise<void>;
abstract scrollLeft(distance?: number): Promise<void>;
abstract scrollUntilTop(startingPoint?: Point): Promise<void>;
abstract scrollUntilBottom(startingPoint?: Point): Promise<void>;
abstract scrollUntilLeft(startingPoint?: Point): Promise<void>;
abstract scrollUntilRight(startingPoint?: Point): Promise<void>;
abstract scrollUp(distance?: number, startingPoint?: Point): Promise<void>;
abstract scrollDown(distance?: number, startingPoint?: Point): Promise<void>;
abstract scrollLeft(distance?: number, startingPoint?: Point): Promise<void>;
abstract scrollRight(distance?: number): Promise<void>;

abstract _forceUsePageContext?(): Promise<WebUIContext>;
Expand Down
17 changes: 9 additions & 8 deletions packages/web-integration/src/playground/static-page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
type WebUIContext,
} from '@/common/utils';
import type { AbstractPage } from '@/page';
import type { Point } from '@midscene/core/.';

const ThrowNotImplemented: any = (methodName: string) => {
throw new Error(
Expand Down Expand Up @@ -39,35 +40,35 @@ export default class StaticPage implements AbstractPage {
return this.uiContext.url;
}

async scrollUntilTop() {
async scrollUntilTop(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilTop');
}

async scrollUntilBottom() {
async scrollUntilBottom(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilBottom');
}

async scrollUntilLeft() {
async scrollUntilLeft(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilLeft');
}

async scrollUntilRight() {
async scrollUntilRight(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilRight');
}

async scrollUp(distance?: number) {
async scrollUp(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollUp');
}

async scrollDown(distance?: number) {
async scrollDown(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollDown');
}

async scrollLeft(distance?: number) {
async scrollLeft(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollLeft');
}

async scrollRight(distance?: number) {
async scrollRight(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollRight');
}

Expand Down
Loading
Loading