feat: update the Azure OpenAI integration #191

apps/site/docs/en/model-provider.md

@@ -30,6 +30,14 @@ export MIDSCENE_OPENAI_INIT_CONFIG_JSON='{"baseURL":"....","defaultHeaders":{"ke
 export MIDSCENE_OPENAI_SOCKS_PROXY="socks5://127.0.0.1:1080"
 ```
 
+Using Azure OpenAI Service:
+
+```bash
+export MIDSCENE_USE_AZURE_OPENAI=1
+export MIDSCENE_AZURE_OPENAI_SCOPE="https://cognitiveservices.azure.com/.default"
+export MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON='{"apiVersion": "2024-11-01-preview", "endpoint": "...", "deployment": "..."}'
+```
+
 Note:
 
 - Always choose a model that supports vision input. 

apps/site/docs/zh/model-provider.md

@@ -17,9 +17,6 @@ export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
 # 可选, 如果你想更换 base URL
 export OPENAI_BASE_URL="https://..."
 
-# 可选, 如果你想使用 Azure OpenAI 服务
-export OPENAI_USE_AZURE="true"
-
 # 可选, 如果你想指定模型名称
 export MIDSCENE_MODEL_NAME='qwen-vl-max-lates';
 
@@ -30,6 +27,14 @@ export MIDSCENE_OPENAI_INIT_CONFIG_JSON='{"baseURL":"....","defaultHeaders":{"ke
 export MIDSCENE_OPENAI_SOCKS_PROXY="socks5://127.0.0.1:1080"
 ```
 
+使用 Azure OpenAI 服务时的配置：
+
+```bash
+export MIDSCENE_USE_AZURE_OPENAI=1
+export MIDSCENE_AZURE_OPENAI_SCOPE="https://cognitiveservices.azure.com/.default"
+export MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON='{"apiVersion": "2024-11-01-preview", "endpoint": "...", "deployment": "..."}'
+```
+
 说明：
 
 - 务必选择一个支持视觉输入的模型。目前我们已知支持的模型有：`gpt-4o`, `qwen-vl-max-latest` (千问), `gemini-1.5-pro`

packages/midscene/package.json

@@ -37,6 +37,7 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
+    "@azure/identity": "4.5.0",
     "@midscene/shared": "workspace:*",
     "openai": "4.57.1",
     "optional": "0.1.4",

packages/midscene/src/ai-model/openai/index.ts

@@ -1,16 +1,23 @@
 import assert from 'node:assert';
 import { AIResponseFormat, type AIUsageInfo } from '@/types';
+import {
+  DefaultAzureCredential,
+  getBearerTokenProvider,
+} from '@azure/identity';
 import { ifInBrowser } from '@midscene/shared/utils';
 import OpenAI, { AzureOpenAI } from 'openai';
 import type { ChatCompletionMessageParam } from 'openai/resources';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import {
+  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
+  MIDSCENE_AZURE_OPENAI_SCOPE,
   MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG,
   MIDSCENE_DEBUG_AI_PROFILE,
   MIDSCENE_LANGSMITH_DEBUG,
   MIDSCENE_MODEL_NAME,
   MIDSCENE_OPENAI_INIT_CONFIG_JSON,
   MIDSCENE_OPENAI_SOCKS_PROXY,
+  MIDSCENE_USE_AZURE_OPENAI,
   OPENAI_API_KEY,
   OPENAI_BASE_URL,
   OPENAI_USE_AZURE,
@@ -26,6 +33,7 @@ import { assertSchema } from '../prompt/util';
 export function preferOpenAIModel(preferVendor?: 'coze' | 'openAI') {
   if (preferVendor && preferVendor !== 'openAI') return false;
   if (getAIConfig(OPENAI_API_KEY)) return true;
+  if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) return true;
 
   return Boolean(getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON));
 }
@@ -47,14 +55,37 @@ async function createOpenAI() {
 
   const socksProxy = getAIConfig(MIDSCENE_OPENAI_SOCKS_PROXY);
   const socksAgent = socksProxy ? new SocksProxyAgent(socksProxy) : undefined;
+
   if (getAIConfig(OPENAI_USE_AZURE)) {
+    // this is deprecated
     openai = new AzureOpenAI({
       baseURL: getAIConfig(OPENAI_BASE_URL),
       apiKey: getAIConfig(OPENAI_API_KEY),
       httpAgent: socksAgent,
       ...extraConfig,
       dangerouslyAllowBrowser: true,
     });
+  } else if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) {
+    // sample code: https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/openai/openai/samples/cookbook/simpleCompletionsPage/app.js
+    const scope = getAIConfig(MIDSCENE_AZURE_OPENAI_SCOPE);
+
+    assert(
+      !ifInBrowser,
+      'Azure OpenAI is not supported in browser with Midscene.',
+    );
+    const credential = new DefaultAzureCredential();
+
+    assert(scope, 'MIDSCENE_AZURE_OPENAI_SCOPE is required');
+    const tokenProvider = getBearerTokenProvider(credential, scope);
+
+    const extraAzureConfig = getAIConfigInJson(
+      MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
+    );
+    openai = new AzureOpenAI({
+      azureADTokenProvider: tokenProvider,
+      ...extraConfig,
+      ...extraAzureConfig,
+    });
   } else {
     openai = new OpenAI({
       baseURL: getAIConfig(OPENAI_BASE_URL),

packages/midscene/src/env.ts

@@ -11,11 +11,19 @@ export const MIDSCENE_OPENAI_SOCKS_PROXY = 'MIDSCENE_OPENAI_SOCKS_PROXY';
 export const OPENAI_API_KEY = 'OPENAI_API_KEY';
 export const OPENAI_BASE_URL = 'OPENAI_BASE_URL';
 export const MIDSCENE_MODEL_TEXT_ONLY = 'MIDSCENE_MODEL_TEXT_ONLY';
-export const OPENAI_USE_AZURE = 'OPENAI_USE_AZURE';
+
 export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
 export const MATCH_BY_POSITION = 'MATCH_BY_POSITION';
 export const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME';
 
+export const MIDSCENE_USE_AZURE_OPENAI = 'MIDSCENE_USE_AZURE_OPENAI';
+export const MIDSCENE_AZURE_OPENAI_SCOPE = 'MIDSCENE_AZURE_OPENAI_SCOPE';
+export const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON =
+  'MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON';
+
+// @deprecated
+export const OPENAI_USE_AZURE = 'OPENAI_USE_AZURE';
+
 const allConfigFromEnv = () => {
   return {
     [MIDSCENE_OPENAI_INIT_CONFIG_JSON]:
@@ -39,6 +47,13 @@ const allConfigFromEnv = () => {
       process.env[MIDSCENE_REPORT_TAG_NAME] || undefined,
     [MIDSCENE_OPENAI_SOCKS_PROXY]:
       process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || undefined,
+    [MIDSCENE_USE_AZURE_OPENAI]:
+      process.env[MIDSCENE_USE_AZURE_OPENAI] || undefined,
+    [MIDSCENE_AZURE_OPENAI_SCOPE]:
+      process.env[MIDSCENE_AZURE_OPENAI_SCOPE] ||
+      'https://cognitiveservices.azure.com/.default',
+    [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]:
+      process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || undefined,
   };
 };