Merge branch 'next' of github.com:firebase/genkit into fix-types

firebase · Oct 22, 2024 · 7efc9f0 · 7efc9f0
2 parents 5032edb + f6e8982
commit 7efc9f0
Show file tree

Hide file tree

Showing 107 changed files with 2,329 additions and 2,567 deletions.
diff --git a/README.md b/README.md
@@ -88,9 +88,14 @@ Find excellent examples of community-built plugins for OpenAI, Anthropic, Cohere
 
 ## Try Genkit on IDX
 
-<img src="docs/resources/idx-logo.png" width="50" alt="Project IDX logo">
-
-Want to try Genkit without a local setup? [Explore it on Project IDX](https://idx.google.com/new/genkit), Google's AI-assisted workspace for full-stack app development in the cloud.
+Want to skip the local setup? Click below to try out Genkit using [Project IDX](https://idx.dev), Google's AI-assisted workspace for full-stack app development in the cloud.
+
+<a href="https://idx.google.com/import?url=">
+  <img
+    height="32"
+    alt="Try in IDX"
+    src="https://cdn.idx.dev/btn/try_purple_32.svg">
+</a>
 
 ## Sample apps
 

diff --git a/docs/plugins/google-cloud.md b/docs/plugins/google-cloud.md
@@ -221,32 +221,84 @@ Common dimensions include:
 - `topK` - the inference topK [value](https://ai.google.dev/docs/concepts#model-parameters).
 - `topP` - the inference topP [value](https://ai.google.dev/docs/concepts#model-parameters).
 
-### Flow-level metrics
+### Feature-level metrics
+
+Features are the top-level entry-point to your Genkit code. In most cases, this
+will be a flow, but if you do not use flows, this will be the top-most span in a trace.
+
+| Name                    | Type      | Description             |
+| ----------------------- | --------- | ----------------------- |
+| genkit/feature/requests | Counter   | Number of requests      |
+| genkit/feature/latency  | Histogram | Execution latency in ms |
+
+Each feature-level metric contains the following dimensions:
+
+| Name          | Description                                                                      |
+| ------------- | -------------------------------------------------------------------------------- |
+| name          | The name of the feature. In most cases, this is the top-level Genkit flow        |
+| status        | 'success' or 'failure' depending on whether or not the feature request succeeded |
+| error         | Only set when `status=failure`. Contains the error type that caused the failure  |
+| source        | The Genkit source language. Eg. 'ts'                                             |
+| sourceVersion | The Genkit framework version                                                     |
 
-| Name                 | Dimensions                           |
-| -------------------- | ------------------------------------ |
-| genkit/flow/requests | flow_name, error_code, error_message |
-| genkit/flow/latency  | flow_name                            |
 
 ### Action-level metrics
 
-| Name                   | Dimensions                           |
-| ---------------------- | ------------------------------------ |
-| genkit/action/requests | flow_name, error_code, error_message |
-| genkit/action/latency  | flow_name                            |
+Actions represent a generic step of execution within Genkit. Each of these steps
+will have the following metrics tracked:
+
+| Name                    | Type      | Description                                   |
+| ----------------------- | --------- | --------------------------------------------- |
+| genkit/action/requests  | Counter   | Number of times this action has been executed |
+| genkit/action/latency   | Histogram | Execution latency in ms                       |
+
+Each action-level metric contains the following dimensions:
+
+| Name          | Description                                                                                          |
+| ------------- | ---------------------------------------------------------------------------------------------------- |
+| name          | The name of the action                                                                               |
+| featureName   | The name of the parent feature being executed                                                        |
+| path          | The path of execution from the feature root to this action. eg. '/myFeature/parentAction/thisAction' |
+| status        | 'success' or 'failure' depending on whether or not the action succeeded                              |
+| error         | Only set when `status=failure`. Contains the error type that caused the failure                      |
+| source        | The Genkit source language. Eg. 'ts'                                                                 |
+| sourceVersion | The Genkit framework version                                                                         |
 
 ### Generate-level metrics
 
-| Name                                 | Dimensions                                                           |
-| ------------------------------------ | -------------------------------------------------------------------- |
-| genkit/ai/generate                   | flow_path, model, temperature, topK, topP, error_code, error_message |
-| genkit/ai/generate/input_tokens      | flow_path, model, temperature, topK, topP                            |
-| genkit/ai/generate/output_tokens     | flow_path, model, temperature, topK, topP                            |
-| genkit/ai/generate/input_characters  | flow_path, model, temperature, topK, topP                            |
-| genkit/ai/generate/output_characters | flow_path, model, temperature, topK, topP                            |
-| genkit/ai/generate/input_images      | flow_path, model, temperature, topK, topP                            |
-| genkit/ai/generate/output_images     | flow_path, model, temperature, topK, topP                            |
-| genkit/ai/generate/latency           | flow_path, model, temperature, topK, topP, error_code, error_message |
+These are special action metrics relating to actions that interact with a model.
+In addition to requests and latency, input and output are also tracked, with model
+specific dimensions that make debugging and configuration tuning easier.
+
+| Name                                 | Type      | Description                                |
+| ------------------------------------ | --------- | ------------------------------------------ |
+| genkit/ai/generate/requests          | Counter   | Number of times this model has been called |
+| genkit/ai/generate/latency           | Histogram | Execution latency in ms                    |
+| genkit/ai/generate/input/tokens      | Counter   | Input tokens                               |
+| genkit/ai/generate/output/tokens     | Counter   | Output tokens                              |
+| genkit/ai/generate/input/characters  | Counter   | Input characters                           |
+| genkit/ai/generate/output/characters | Counter   | Output characters                          |
+| genkit/ai/generate/input/images      | Counter   | Input images                               |
+| genkit/ai/generate/output/images     | Counter   | Output images                              |
+| genkit/ai/generate/input/audio       | Counter   | Input audio files                          |
+| genkit/ai/generate/output/audio      | Counter   | Output audio files                         |
+
+Each generate-level metric contains the following dimensions:
+
+| Name            | Description                                                                                          |
+| --------------- | ---------------------------------------------------------------------------------------------------- |
+| modelName       | The name of the model                                                                                |
+| featureName     | The name of the parent feature being executed                                                        |
+| path            | The path of execution from the feature root to this action. eg. '/myFeature/parentAction/thisAction' |
+| temperature     | The temperature parameter passed to the model                                                        |
+| maxOutputTokens | The maxOutputTokens parameter passed to the model                                                    |
+| topK            | The topK parameter passed to the model                                                               |
+| topP            | The topP parameter passed to the model                                                               |
+| latencyMs       | The response time taken by the model                                                                 |
+| status          | 'success' or 'failure' depending on whether or not the feature request succeeded                     |
+| error           | Only set when `status=failure`. Contains the error type that caused the failure                      |
+| source          | The Genkit source language. Eg. 'ts'                                                                 |
+| sourceVersion   | The Genkit framework version                                                                         |
 
 Visualizing metrics can be done through the Metrics Explorer. Using the side menu, select 'Logging' and click 'Metrics explorer'
 

diff --git a/genkit-tools/cli/package.json b/genkit-tools/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "genkit-cli",
-  "version": "0.6.0-dev.2",
+  "version": "0.9.0-dev.1",
   "description": "CLI for interacting with the Google Genkit AI framework",
   "license": "Apache-2.0",
   "keywords": [
@@ -28,7 +28,7 @@
   "dependencies": {
     "@genkit-ai/tools-common": "workspace:*",
     "@genkit-ai/telemetry-server": "workspace:*",
-    "axios": "^1.6.7",
+    "axios": "^1.7.7",
     "colorette": "^2.0.20",
     "commander": "^11.1.0",
     "extract-zip": "^2.0.1",

diff --git a/genkit-tools/common/package.json b/genkit-tools/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@genkit-ai/tools-common",
-  "version": "0.6.0-dev.2",
+  "version": "0.9.0-dev.1",
   "scripts": {
     "compile": "tsc -b ./tsconfig.cjs.json ./tsconfig.esm.json ./tsconfig.types.json",
     "build:clean": "rimraf ./lib",
@@ -12,7 +12,7 @@
     "@asteasolutions/zod-to-openapi": "^7.0.0",
     "@trpc/server": "10.45.0",
     "adm-zip": "^0.5.12",
-    "axios": "^1.6.7",
+    "axios": "^1.7.7",
     "body-parser": "^1.20.2",
     "chokidar": "^3.5.3",
     "colorette": "^2.0.20",

diff --git a/genkit-tools/common/src/eval/evaluate.ts b/genkit-tools/common/src/eval/evaluate.ts
@@ -216,11 +216,12 @@ async function bulkRunAction(params: {
         testCaseId: c.testCaseId ?? generateTestCaseId(),
       }));
 
+  let states: InferenceRunState[] = [];
   let evalInputs: EvalInput[] = [];
   for (const testCase of testCases) {
-    logger.info(`Running '${actionRef}' ...`);
+    logger.info(`Running inference '${actionRef}' ...`);
     if (isModelAction) {
-      evalInputs.push(
+      states.push(
         await runModelAction({
           manager,
           actionRef,
@@ -229,7 +230,7 @@ async function bulkRunAction(params: {
         })
       );
     } else {
-      evalInputs.push(
+      states.push(
         await runFlowAction({
           manager,
           actionRef,
@@ -239,6 +240,11 @@ async function bulkRunAction(params: {
       );
     }
   }
+
+  logger.info(`Gathering evalInputs...`);
+  for (const state of states) {
+    evalInputs.push(await gatherEvalInput({ manager, actionRef, state }));
+  }
   return evalInputs;
 }
 
@@ -247,7 +253,7 @@ async function runFlowAction(params: {
   actionRef: string;
   testCase: TestCase;
   auth?: any;
-}): Promise<EvalInput> {
+}): Promise<InferenceRunState> {
   const { manager, actionRef, testCase, auth } = { ...params };
   let state: InferenceRunState;
   try {
@@ -274,15 +280,15 @@ async function runFlowAction(params: {
       evalError: `Error when running inference. Details: ${e?.message ?? e}`,
     };
   }
-  return gatherEvalInput({ manager, actionRef, state });
+  return state;
 }
 
 async function runModelAction(params: {
   manager: RuntimeManager;
   actionRef: string;
   testCase: TestCase;
   modelConfig?: any;
-}): Promise<EvalInput> {
+}): Promise<InferenceRunState> {
   const { manager, actionRef, modelConfig, testCase } = { ...params };
   let state: InferenceRunState;
   try {
@@ -304,7 +310,7 @@ async function runModelAction(params: {
       evalError: `Error when running inference. Details: ${e?.message ?? e}`,
     };
   }
-  return gatherEvalInput({ manager, actionRef, state });
+  return state;
 }
 
 async function gatherEvalInput(params: {

diff --git a/genkit-tools/package.json b/genkit-tools/package.json
@@ -23,5 +23,5 @@
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.4"
   },
-  "packageManager": "[email protected].0+sha256.a61b67ff6cc97af864564f4442556c22a04f2e5a7714fbee76a1011361d9b726"
+  "packageManager": "[email protected].2+sha256.2ef6e547b0b07d841d605240dce4d635677831148cd30f6d564b8f4f928f73d2"
 }
diff --git a/genkit-tools/pnpm-lock.yaml b/genkit-tools/pnpm-lock.yaml
diff --git a/genkit-tools/telemetry-server/package.json b/genkit-tools/telemetry-server/package.json
@@ -7,7 +7,7 @@
     "genai",
     "generative-ai"
   ],
-  "version": "0.6.0-dev.2",
+  "version": "0.9.0-dev.1",
   "type": "commonjs",
   "scripts": {
     "compile": "tsc -b ./tsconfig.cjs.json ./tsconfig.esm.json ./tsconfig.types.json",

diff --git a/js/ai/package.json b/js/ai/package.json
@@ -7,7 +7,7 @@
     "genai",
     "generative-ai"
   ],
-  "version": "0.6.0-dev.2",
+  "version": "0.9.0-dev.1",
   "type": "commonjs",
   "scripts": {
     "check": "tsc",