|
| 1 | +import { Value } from "@sinclair/typebox/value"; |
1 | 2 | import Decimal from "decimal.js"; |
| 3 | +import { encodingForModel, Tiktoken } from "js-tiktoken"; |
2 | 4 | import OpenAI from "openai"; |
| 5 | +import { commentEnum, CommentType } from "../configuration/comment-types"; |
3 | 6 | import configuration from "../configuration/config-reader"; |
4 | 7 | import { OPENAI_API_KEY } from "../configuration/constants"; |
5 | 8 | import { |
6 | 9 | ContentEvaluatorConfiguration, |
7 | 10 | contentEvaluatorConfigurationType, |
8 | 11 | } from "../configuration/content-evaluator-config"; |
9 | | -import { IssueActivity } from "../issue-activity"; |
10 | | -import { GithubCommentScore, Module, Result } from "./processor"; |
11 | | -import { Value } from "@sinclair/typebox/value"; |
12 | | -import { commentEnum, CommentType } from "../configuration/comment-types"; |
13 | 12 | import logger from "../helpers/logger"; |
| 13 | +import { IssueActivity } from "../issue-activity"; |
14 | 14 | import openAiRelevanceResponseSchema, { RelevancesByOpenAi } from "../types/openai-type"; |
| 15 | +import { GithubCommentScore, Module, Result } from "./processor"; |
15 | 16 |
|
16 | 17 | /** |
17 | 18 | * Evaluates and rates comments. |
@@ -112,30 +113,47 @@ export class ContentEvaluatorModule implements Module { |
112 | 113 | return commentsWithScore; |
113 | 114 | } |
114 | 115 |
|
| 116 | + /** |
| 117 | + * Will try to predict the maximum of tokens expected, to a maximum of totalTokenLimit. |
| 118 | + */ |
| 119 | + _calculateMaxTokens(prompt: string, totalTokenLimit: number = 16384) { |
| 120 | + const tokenizer: Tiktoken = encodingForModel("gpt-4o-2024-08-06"); |
| 121 | + const inputTokens = tokenizer.encode(prompt).length; |
| 122 | + return Math.min(inputTokens, totalTokenLimit); |
| 123 | + } |
| 124 | + |
| 125 | + _generateDummyResponse(comments: { id: number; comment: string }[]) { |
| 126 | + return comments.reduce<Record<string, number>>((acc, curr) => { |
| 127 | + return { ...acc, [curr.id]: 0.5 }; |
| 128 | + }, {}); |
| 129 | + } |
| 130 | + |
115 | 131 | async _evaluateComments( |
116 | 132 | specification: string, |
117 | 133 | comments: { id: number; comment: string }[] |
118 | 134 | ): Promise<RelevancesByOpenAi> { |
119 | 135 | const prompt = this._generatePrompt(specification, comments); |
| 136 | + const dummyResponse = JSON.stringify(this._generateDummyResponse(comments), null, 2); |
| 137 | + const maxTokens = this._calculateMaxTokens(dummyResponse); |
120 | 138 |
|
121 | 139 | const response: OpenAI.Chat.ChatCompletion = await this._openAi.chat.completions.create({ |
122 | | - model: "gpt-4o", |
| 140 | + model: "gpt-4o-2024-08-06", |
123 | 141 | response_format: { type: "json_object" }, |
124 | 142 | messages: [ |
125 | 143 | { |
126 | 144 | role: "system", |
127 | 145 | content: prompt, |
128 | 146 | }, |
129 | 147 | ], |
130 | | - temperature: 1, |
131 | | - max_tokens: 128, |
| 148 | + max_tokens: maxTokens, |
132 | 149 | top_p: 1, |
| 150 | + temperature: 1, |
133 | 151 | frequency_penalty: 0, |
134 | 152 | presence_penalty: 0, |
135 | 153 | }); |
136 | 154 |
|
137 | 155 | const rawResponse = String(response.choices[0].message.content); |
138 | | - logger.info(`OpenAI raw response: ${rawResponse}`); |
| 156 | + logger.info(`OpenAI raw response (using max_tokens: ${maxTokens}): ${rawResponse}`); |
139 | 157 |
|
140 | 158 | const jsonResponse = JSON.parse(rawResponse); |
141 | 159 |
|
|
0 commit comments