diff --git a/composer.json b/composer.json
index 2b06f634..ff6e2d2d 100644
--- a/composer.json
+++ b/composer.json
@@ -34,6 +34,7 @@
"ext-dom": "*",
"cebe/markdown": "^1.2",
"duzun/hquery": "^3.1",
+ "eftec/bladeone": "^4.16",
"gioni06/gpt3-tokenizer": "^1.2",
"guzzlehttp/psr7": "^2.7",
"illuminate/database": "^11.10",
@@ -49,7 +50,6 @@
"spatie/array-to-xml": "^3.3",
"spatie/browsershot": "^4.1",
"spatie/php-structure-discoverer": "^2.1",
- "spatie/yaml-front-matter": "^2.0",
"symfony/browser-kit": "^7.1",
"symfony/css-selector": "^7.1",
"symfony/dom-crawler": "^7.1",
@@ -58,7 +58,8 @@
"symfony/var-dumper": "^6.4 || ^7.0",
"toolkit/cli-utils": "^2.0",
"twig/twig": "^3.0",
- "vimeo/psalm": "dev-master"
+ "vimeo/psalm": "dev-master",
+ "webuni/front-matter": "^2.0"
},
"config": {
"allow-plugins": {
@@ -70,6 +71,7 @@
"require": {
"php": "^8.2",
"ext-fileinfo": "*",
+ "ext-simplexml": "*",
"adbario/php-dot-notation": "^3.3",
"aimeos/map": "^3.8",
"guzzlehttp/guzzle": "^7.8",
@@ -77,6 +79,7 @@
"phpstan/phpdoc-parser": "^1.29",
"psr/event-dispatcher": "^1.0",
"psr/log": "^3.0",
+ "spatie/yaml-front-matter": "^2.0",
"symfony/intl": "^7.1",
"symfony/property-access": "^6.4 || ^7.0",
"symfony/property-info": "^6.4 || ^7.0",
diff --git a/config/prompt.php b/config/prompt.php
new file mode 100644
index 00000000..757831f0
--- /dev/null
+++ b/config/prompt.php
@@ -0,0 +1,26 @@
+ 'twig',
+
+ 'settings' => [
+ 'twig' => [
+ 'templateType' => 'twig',
+ 'resourcePath' => '/../../../../prompts/twig',
+ 'cachePath' => '/tmp/instructor/cache/twig',
+ 'extension' => '.twig',
+ 'frontMatterTags' => ['{#---', '---#}'],
+ 'frontMatterFormat' => 'yaml',
+ 'metadata' => [
+ 'autoReload' => true,
+ ],
+ ],
+ 'blade' => [
+ 'templateType' => 'blade',
+ 'resourcePath' => '/../../../../prompts/blade',
+ 'cachePath' => '/tmp/instructor/cache/blade',
+ 'extension' => '.blade.php',
+ 'frontMatterTags' => ['{{--', '--}}'],
+ 'frontMatterFormat' => 'yaml',
+ ],
+ ]
+];
diff --git a/docs/advanced/prompts.mdx b/docs/advanced/prompts.mdx
new file mode 100644
index 00000000..28d3b3da
--- /dev/null
+++ b/docs/advanced/prompts.mdx
@@ -0,0 +1,573 @@
+## Prompts
+
+`Prompt` class in Instructor provides a powerful and flexible
+way to manage prompts for LLM interactions. It supports multiple
+template engines (Twig, Blade), front matter, variable injection,
+and validation.
+
+The Prompt class helps manage:
+- Template-based prompts using Twig or Blade engines
+- Front matter for metadata and schema definitions
+- Variable injection and validation
+- Chat message formatting
+- Prompt validation and error checking
+
+
+
+
+
+### Quick Start
+
+Use default settings to create a prompt:
+
+```php
+use Cognesy\Instructor\Extras\Prompt\Prompt;
+
+// Create a simple prompt with variables
+$prompt = Prompt::text('hello', ['name' => 'World']);
+// Output: "Hello, World!"
+
+// Create a prompt with chat messages
+$messages = Prompt::messages('capital', ['country' => 'France']);
+// Output: Array of chat messages asking about France's capital
+```
+
+Both 'hello' and 'capital' prompts have been defined in the prompts directory,
+in '/prompts/twig/hello.twig' and '/prompts/twig/capital.twig' files respectively.
+
+`Prompt` class will use the default template engine to render
+them to text and chat messages.
+
+Default setting is defined in the `/config/prompt.php` file in the root
+directory of Instructor. Default setting uses Twig template engine, but
+you can change it to Blade (or, in the future, other supported engine).
+
+
+
+
+### Supported Template Engines
+
+#### Twig Templates
+
+Twig templates offer powerful templating features with clean syntax:
+
+```twig
+{# prompts/twig/hello.twig #}
+{#---
+description: Hello world template
+variables:
+ name:
+ description: Name to greet
+ type: string
+ default: World
+---#}
+
+Hello, {{ name }}!
+```
+See [Twig documentation](https://twig.symfony.com/doc/3.x/templates.html) for more details.
+
+
+
+#### Blade Templates
+
+Blade templates provide Laravel-style syntax:
+
+```php
+{{-- prompts/blade/hello.blade.php --}}
+{{--
+description: Hello world template
+variables:
+ name:
+ description: Name to greet
+ type: string
+ default: World
+--}}
+
+Hello, {{ $name }}!
+```
+See [Blade documentation](https://laravel.com/docs/11.x/blade) for more details.
+
+> NOTE: Instructor Prompt uses BladeOne engine, which is a standalone version
+> of Laravel Blade engine. Be aware that some features may differ from Laravel
+> Blade, but the syntax is mostly the same.
+
+
+
+
+
+### Configuration
+
+The prompt engine in Instructor can be configured using a configuration
+file or programmatically. This guide explains the configuration options
+and how to use them effectively.
+
+#### Custom Configuration Location
+
+You can specify a custom configuration location using environment variables:
+
+```bash
+INSTRUCTOR_CONFIG_PATH=/path/to/config
+```
+
+#### Basic Configuration
+
+Create a configuration file for your prompts:
+
+```php
+use Cognesy\Instructor\Extras\Prompt\Data\PromptEngineConfig;
+use Cognesy\Instructor\Extras\Prompt\Enums\TemplateType;
+
+$config = new PromptEngineConfig(
+ templateType: TemplateType::Twig,
+ resourcePath: '/prompts/twig',
+ cachePath: '/cache/prompts',
+ extension: '.twig'
+);
+
+$prompt = new Prompt(config: $config);
+```
+
+#### Loading Configuration from Settings
+
+```php
+$prompt = Prompt::using('default'); // Loads default settings
+```
+
+
+
+
+
+### Using Prompts
+
+#### Loading Prompts from Files
+
+```php
+// Load a prompt template
+$prompt = Prompt::get('hello');
+
+// Render with variables
+$result = $prompt->withValues(['name' => 'John'])->toText();
+// Output: "Hello, John!"
+```
+
+#### Creating Prompts from Strings
+
+```php
+$prompt = new Prompt();
+$content = "Hello, {{ name }}!";
+$result = $prompt
+ ->withTemplateContent($content)
+ ->withValues(['name' => 'Jane'])
+ ->toText();
+// Output: "Hello, Jane!"
+```
+
+#### Rendering Chat Messages
+
+```php
+// Using a chat template
+$prompt = Prompt::get('capital');
+$messages = $prompt
+ ->withValues(['country' => 'France'])
+ ->toMessages();
+// Output: ChatMessages object with system/user/assistant messages
+```
+
+
+
+
+
+
+### Prompt Info (front matter)
+
+PromptInfo is a front matter that allows you to define metadata and schema for your prompts:
+
+```twig
+{#---
+description: Capital finder template
+variables:
+ country:
+ description: Country name
+ type: string
+ default: France
+schema:
+ name: capital
+ properties:
+ name:
+ description: Capital city name
+ type: string
+ required: [name]
+---#}
+
+ You are a helpful assistant.
+ What is the capital of {{ country }}?
+
+```
+
+#### Accessing PromptInfo Data
+
+```php
+$prompt = Prompt::get('capital');
+$info = $prompt->info();
+
+echo $info->field('description');
+// Output: "Capital finder template"
+
+$variables = $info->variables();
+// Output: Array of variable definitions
+```
+
+
+
+
+
+
+### Working with Variables
+
+#### Defining Variables
+
+Variables can be defined in front matter:
+
+```yaml
+variables:
+ name:
+ description: User's name
+ type: string
+ default: Guest
+ age:
+ description: User's age
+ type: integer
+```
+
+#### Injecting Variables
+
+```php
+$prompt = Prompt::get('user_info');
+$result = $prompt->withValues([
+ 'name' => 'John',
+ 'age' => 25
+])->toText();
+```
+
+
+
+
+
+
+
+### Validation
+
+#### Validating Prompts
+
+The Prompt class includes built-in validation of prompts content:
+
+```php
+$prompt = Prompt::get('user_info');
+$errors = $prompt->validationErrors();
+
+if (!empty($errors)) {
+ foreach ($errors as $error) {
+ echo "Validation error: $error\n";
+ }
+}
+```
+
+#### Variable Validation
+
+```php
+// Check if all required variables are provided
+$prompt = Prompt::get('user_info');
+$prompt = $prompt->withValues(['name' => 'John']);
+$errors = $prompt->validationErrors();
+// Will show error if 'age' is required but not provided
+```
+
+
+
+
+
+
+### Best Practices
+
+1. **Organization**: Keep prompts in dedicated directories based on template engine
+```
+prompts/
+├── twig/
+│ ├── user_info.twig
+│ └── capital.twig
+└── blade/
+ ├── user_info.blade.php
+ └── capital.blade.php
+ ```
+
+2. **Front Matter**: Always include descriptions and variable definitions
+```yaml
+description: Clear description of the prompt's purpose
+variables:
+ name:
+ description: Clear description of the variable
+ type: string
+ default: Default value if applicable
+ ```
+
+3. **Validation**: Always validate prompts before using them in production
+```php
+$prompt = Prompt::get('user_info');
+if (!empty($prompt->validationErrors())) {
+ throw new Exception('Prompt validation errors: ' . implode(', ', $prompt->validationErrors()));
+}
+ ```
+
+4. **Caching**: Use cache paths in configuration for better performance
+```php
+$config = new PromptEngineConfig(
+ cachePath: '/path/to/cache'
+);
+ ```
+
+
+
+
+
+### Example: Complete Usage
+
+Here's a complete example showing multiple features:
+
+```php
+// Define a prompt template
+$template = <<
+ You are a friendly assistant.
+
+ Hello! My name is {{ name }} and I like {{ preferences|join(', ') }}.
+
+
+TWIG;
+
+// Create and configure prompt
+$prompt = new Prompt();
+$prompt->withTemplateContent($template)
+ ->withValues([
+ 'name' => 'Alice',
+ 'preferences' => ['reading', 'hiking', 'photography']
+ ]);
+
+// Validate
+if (!empty($prompt->validationErrors())) {
+ throw new Exception('Invalid prompt configuration');
+}
+
+// Get chat messages
+$messages = $prompt->toMessages();
+
+// Use with Instructor
+$response = (new Instructor)->respond(
+ messages: $messages,
+ responseModel: YourResponseModel::class
+);
+```
+
+
+
+
+
+### Configuration File
+
+By default, Instructor looks for prompt configuration in `config/prompt.php`.
+Here's how to set up and customize your prompt engine:
+
+```php
+ 'twig',
+
+ // Available configuration settings
+ 'settings' => [
+ 'twig' => [
+ 'templateType' => 'twig',
+ 'resourcePath' => '/../../../../prompts/twig',
+ 'cachePath' => '/tmp/instructor/cache/twig',
+ 'extension' => '.twig',
+ 'frontMatterTags' => ['{#---', '---#}'],
+ 'frontMatterFormat' => 'yaml',
+ 'metadata' => [
+ 'autoReload' => true,
+ ],
+ ],
+ 'blade' => [
+ 'templateType' => 'blade',
+ 'resourcePath' => '/../../../../prompts/blade',
+ 'cachePath' => '/tmp/instructor/cache/blade',
+ 'extension' => '.blade.php',
+ 'frontMatterTags' => ['{{--', '--}}'],
+ 'frontMatterFormat' => 'yaml',
+ ],
+ ]
+];
+```
+
+#### Configuration Options
+
+Each configuration setting supports the following options:
+
+##### Core Settings
+
+| Option | Description | Example |
+|--------|-------------|---------|
+| `templateType` | Template engine to use ('twig' or 'blade') | `'twig'` |
+| `resourcePath` | Path to prompt template files | `'/prompts/twig'` |
+| `cachePath` | Path for compiled templates | `'/tmp/cache/twig'` |
+| `extension` | File extension for templates | `'.twig'` |
+
+##### Front Matter Settings
+
+| Option | Description | Example |
+|--------|-------------|---------|
+| `frontMatterTags` | Array of start and end tags for front matter | `['{#---', '---#}']` |
+| `frontMatterFormat` | Format of front matter ('yaml', 'json', 'toml') | `'yaml'` |
+
+##### Engine-Specific Settings
+
+| Option | Description | Example |
+|--------|-------------|---------|
+| `metadata` | Engine-specific configuration | `['autoReload' => true]` |
+
+#### Using Configurations
+
+##### Default Configuration
+
+The default configuration is specified by `defaultSetting` and is used when no specific configuration is provided:
+
+```php
+use Cognesy\Instructor\Extras\Prompt\Prompt;
+
+// Uses default configuration (twig in this case)
+$prompt = new Prompt('hello');
+```
+
+##### Specific Configuration
+
+You can specify which configuration to use:
+
+```php
+// Use Blade configuration
+$prompt = Prompt::using('blade')->withTemplate('hello');
+
+// Or with the constructor
+$prompt = new Prompt('hello', setting: 'blade');
+```
+
+#### Programmatic Configuration
+
+You can also create configuration programmatically:
+
+```php
+use Cognesy\Instructor\Extras\Prompt\Data\PromptEngineConfig;
+use Cognesy\Instructor\Extras\Prompt\Enums\TemplateType;
+use Cognesy\Instructor\Extras\Prompt\Enums\FrontMatterFormat;
+
+$config = new PromptEngineConfig(
+ templateType: TemplateType::Twig,
+ resourcePath: '/path/to/prompts',
+ cachePath: '/path/to/cache',
+ extension: '.twig',
+ frontMatterTags: ['{#---', '---#}'],
+ frontMatterFormat: FrontMatterFormat::Yaml,
+ metadata: [
+ 'autoReload' => true,
+ ]
+);
+
+$prompt = new Prompt(config: $config);
+```
+
+#### Configuration Loading
+
+When Instructor initializes, it follows this process to load configuration:
+
+1. Checks for configuration file at the default location
+2. Loads the default setting specified by `defaultSetting`
+3. Merges any user-provided configuration options
+
+#### Engine-Specific Features
+
+##### Twig Configuration
+
+Twig-specific settings in `metadata`:
+
+```php
+'twig' => [
+ // ... other settings ...
+ 'metadata' => [
+ 'autoReload' => true, // Recompile templates when changed
+ ],
+]
+```
+
+##### Blade Configuration
+
+Blade-specific settings can be added to metadata:
+
+```php
+'blade' => [
+ // ... other settings ...
+ 'metadata' => [
+ 'mode' => 'MODE_AUTO', // BladeOne mode
+ ],
+]
+```
+
+#### Best Practices
+
+1. **Environment-Specific Paths**: Use environment variables for paths:
+
+```php
+'cachePath' => env('PROMPT_CACHE_PATH', '/tmp/cache/prompts'),
+```
+
+2. **Template Organization**: Keep templates organized by engine:
+
+```
+prompts/
+├── twig/
+│ └── templates.twig
+└── blade/
+ └── templates.blade.php
+```
+
+3. **Cache Management**: Implement cache clearing in your deployment:
+
+```php
+// Clear cache programmatically
+$cacheDir = config('prompt.settings.twig.cachePath');
+File::cleanDirectory($cacheDir);
+```
+
+4. **Validation**: Validate configuration during application boot:
+
+```php
+use Cognesy\Instructor\Extras\Prompt\Prompt;
+
+public function boot()
+{
+ $prompt = Prompt::using('twig');
+ if (!is_dir($prompt->config()->resourcePath)) {
+ throw new Exception('Prompt resource path not found');
+ }
+}
+```
+
+This configuration system provides flexibility while maintaining
+ease of use, allowing you to customize the prompt engine's behavior
+to match your application's needs.
diff --git a/docs/essentials/prompts.mdx b/docs/essentials/customize_prompts.mdx
similarity index 100%
rename from docs/essentials/prompts.mdx
rename to docs/essentials/customize_prompts.mdx
diff --git a/docs/mint.json b/docs/mint.json
index db8d0e85..075bfd52 100644
--- a/docs/mint.json
+++ b/docs/mint.json
@@ -74,7 +74,7 @@
"essentials/data_model",
"essentials/scalars",
"essentials/validation",
- "essentials/prompts",
+ "essentials/customize_prompts",
"essentials/demonstrations"
]
},
diff --git a/evals/ComplexExtraction/run.php b/evals/ComplexExtraction/run.php
index bb955bd0..006bc1c3 100644
--- a/evals/ComplexExtraction/run.php
+++ b/evals/ComplexExtraction/run.php
@@ -1,14 +1,14 @@
add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
@@ -35,13 +35,13 @@
),
],
postprocessors: [
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.mean_completeness',
observationKey: 'execution.fractionFound',
params: ['unit' => 'fraction', 'format' => '%.2f'],
method: NumberAggregationMethod::Mean,
),
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.latency_p95',
observationKey: 'execution.timeElapsed',
params: ['percentile' => 95, 'unit' => 'seconds'],
diff --git a/evals/LLMModes/run.php b/evals/LLMModes/run.php
index 084f97f1..10152ba8 100644
--- a/evals/LLMModes/run.php
+++ b/evals/LLMModes/run.php
@@ -5,15 +5,13 @@
use Cognesy\Evals\LLMModes\CompanyEval;
use Cognesy\Instructor\Enums\Mode;
-use Cognesy\Instructor\Extras\Evals\Aggregators\AggregateExperimentObservation;
use Cognesy\Instructor\Extras\Evals\Enums\NumberAggregationMethod;
-use Cognesy\Instructor\Extras\Evals\Evaluators\ArrayMatchEval;
-use Cognesy\Instructor\Extras\Evals\Experiment;
use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceCases;
use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceData;
use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceSchema;
use Cognesy\Instructor\Extras\Evals\Executors\RunInference;
-use Cognesy\Instructor\Utils\Debug\Debug;
+use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Observers\Aggregate\AggregateExperimentObserver;
$data = new InferenceData(
messages: [
@@ -61,7 +59,7 @@
]),
],
postprocessors: [
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.reliability',
observationKey: 'execution.is_correct',
params: ['unit' => 'fraction', 'format' => '%.2f'],
diff --git a/evals/SimpleExtraction/run.php b/evals/SimpleExtraction/run.php
index b1d343ab..2baefe3c 100644
--- a/evals/SimpleExtraction/run.php
+++ b/evals/SimpleExtraction/run.php
@@ -3,14 +3,13 @@
use Cognesy\Evals\SimpleExtraction\Company;
use Cognesy\Evals\SimpleExtraction\CompanyEval;
use Cognesy\Instructor\Enums\Mode;
-use Cognesy\Instructor\Extras\Evals\Aggregators\AggregateExperimentObservation;
use Cognesy\Instructor\Extras\Evals\Enums\NumberAggregationMethod;
-use Cognesy\Instructor\Extras\Evals\Evaluators\ArrayMatchEval;
-use Cognesy\Instructor\Extras\Evals\Experiment;
use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceCases;
use Cognesy\Instructor\Extras\Evals\Executors\Data\InstructorData;
use Cognesy\Instructor\Extras\Evals\Executors\RunInstructor;
-use Cognesy\Instructor\Utils\Debug\Debug;
+use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Observers\Aggregate\AggregateExperimentObserver;
+use Cognesy\Instructor\Extras\Evals\Observers\Evaluate\ArrayMatchEval;
$loader = require 'vendor/autoload.php';
$loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
@@ -39,32 +38,40 @@
expectations: [
'name' => 'ACME',
'year' => 2020
- ]),
- new ArrayMatchEval(expected: [
- 'name' => 'ACME',
- 'year' => 2020,
- ]),
+ ]
+ ),
+ new ArrayMatchEval(
+ expected: [
+ 'name' => 'ACME',
+ 'year' => 2020,
+ ],
+ metricNames: [
+ 'precision' => 'execution.precision',
+ 'recall' => 'execution.recall',
+ 'field_feedback' => 'execution.field_feedback',
+ ]
+ ),
],
postprocessors: [
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.reliability',
observationKey: 'execution.is_correct',
params: ['unit' => 'fraction', 'format' => '%.2f'],
method: NumberAggregationMethod::Mean,
),
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.mean_precision',
observationKey: 'execution.precision',
params: ['unit' => 'fraction', 'format' => '%.2f'],
method: NumberAggregationMethod::Mean,
),
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.mean_recall',
observationKey: 'execution.recall',
params: ['unit' => 'fraction', 'format' => '%.2f'],
method: NumberAggregationMethod::Mean,
),
- new AggregateExperimentObservation(
+ new AggregateExperimentObserver(
name: 'experiment.latency_p95',
observationKey: 'execution.timeElapsed',
params: ['percentile' => 95, 'unit' => 'seconds'],
diff --git a/evals/UseExamples/Company.php b/evals/UseExamples/Company.php
new file mode 100644
index 00000000..b2b644a6
--- /dev/null
+++ b/evals/UseExamples/Company.php
@@ -0,0 +1,12 @@
+key = $key;
+ $this->expectations = $expectations;
+ }
+
+ public function observe(Execution $execution): Observation {
+ $company = $execution->data()->get('response')?->value();
+ $isCorrect = ($this->expectations['name'] === ($company->name ?? null))
+ && ($this->expectations['year'] === ($company->year ?? null));
+
+ return Observation::make(
+ type: 'metric',
+ key: $this->key,
+ value: $isCorrect ? 1 : 0,
+ metadata: [
+ 'executionId' => $execution->id(),
+ 'data' => json_encode($company),
+ ],
+ );
+ }
+}
diff --git a/evals/UseExamples/run.php b/evals/UseExamples/run.php
new file mode 100644
index 00000000..9c07c6a5
--- /dev/null
+++ b/evals/UseExamples/run.php
@@ -0,0 +1,83 @@
+add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
+
+$data = new InstructorData(
+ messages: [
+ ['role' => 'user', 'content' => 'YOUR GOAL: Use tools to store the information from context based on user questions.'],
+ ['role' => 'user', 'content' => 'CONTEXT: Our company ACME was founded in 2020.'],
+ ['role' => 'user', 'content' => 'What is the name and founding year of our company?'],
+ ],
+ responseModel: Company::class,
+);
+
+//Debug::enable();
+
+$experiment = new Experiment(
+ cases: InferenceCases::except(
+ connections: ['ollama'],
+ modes: [Mode::JsonSchema, Mode::Text],
+ stream: [true]
+ ),
+ executor: new RunInstructor($data),
+ processors: [
+ new CompanyEval(
+ key: 'execution.is_correct',
+ expectations: [
+ 'name' => 'ACME',
+ 'year' => 2020
+ ]
+ ),
+ new ArrayMatchEval(
+ expected: [
+ 'name' => 'ACME',
+ 'year' => 2020,
+ ],
+ metricNames: [
+ 'precision' => 'execution.precision',
+ 'recall' => 'execution.recall',
+ 'field_feedback' => 'execution.field_feedback',
+ ]
+ ),
+ ],
+ postprocessors: [
+ new AggregateExperimentObserver(
+ name: 'experiment.reliability',
+ observationKey: 'execution.is_correct',
+ params: ['unit' => 'fraction', 'format' => '%.2f'],
+ method: NumberAggregationMethod::Mean,
+ ),
+ new AggregateExperimentObserver(
+ name: 'experiment.mean_precision',
+ observationKey: 'execution.precision',
+ params: ['unit' => 'fraction', 'format' => '%.2f'],
+ method: NumberAggregationMethod::Mean,
+ ),
+ new AggregateExperimentObserver(
+ name: 'experiment.mean_recall',
+ observationKey: 'execution.recall',
+ params: ['unit' => 'fraction', 'format' => '%.2f'],
+ method: NumberAggregationMethod::Mean,
+ ),
+ new AggregateExperimentObserver(
+ name: 'experiment.latency_p95',
+ observationKey: 'execution.timeElapsed',
+ params: ['percentile' => 95, 'unit' => 'seconds'],
+ method: NumberAggregationMethod::Percentile,
+ ),
+ ],
+);
+
+$outputs = $experiment->execute();
diff --git a/examples/A05_Extras/PromptText/run.php b/examples/A05_Extras/PromptText/run.php
new file mode 100644
index 00000000..2f6d475c
--- /dev/null
+++ b/examples/A05_Extras/PromptText/run.php
@@ -0,0 +1,48 @@
+---
+title: 'Prompts'
+docname: 'prompt_text'
+---
+
+## Overview
+
+`Prompt` class in Instructor PHP provides a way to define and use
+prompt templates using Twig or Blade template syntax.
+
+
+## Example
+
+```php
+add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
+
+use Cognesy\Instructor\Extras\Prompt\Prompt;
+use Cognesy\Instructor\Features\LLM\Inference;
+use Cognesy\Instructor\Utils\Str;
+
+// EXAMPLE 1: Simplfied API
+
+// use default template language, prompt files are in /prompts/twig/.twig
+$text = Prompt::text('capital', ['country' => 'Germany']);
+$answer = (new Inference)->create(messages: $text)->toText();
+
+echo "EXAMPLE 1: prompt = $text\n";
+echo "ASSISTANT: $answer\n";
+echo "\n";
+assert(Str::contains($answer, 'Berlin'));
+
+// EXAMPLE 2: Define prompt template inline
+
+$text = Prompt::using('twig')
+ ->withTemplateContent('What is capital of {{country}}')
+ ->withValues(['country' => 'Germany'])
+ ->toText();
+$answer = (new Inference)->create(messages: $text)->toText();
+
+echo "EXAMPLE 2: prompt = $text\n";
+echo "ASSISTANT: $answer\n";
+echo "\n";
+assert(Str::contains($answer, 'Berlin'));
+
+?>
+```
diff --git a/notes/NOTES.md b/notes/NOTES.md
index 43df5271..7039eec7 100644
--- a/notes/NOTES.md
+++ b/notes/NOTES.md
@@ -4,8 +4,6 @@
- Evals / eval framework
* execution level correctness metric
- * add input, output, etc. tokens default metrics
- * simplify contracts - currently 5 (!) contracts for observations
- Add 'Output' section to each example, generate it and include in docs, so reader can see what they can expect
- Logging via PSR-3
- Schema abstraction layer - decouple names and descriptions from the model
@@ -78,16 +76,6 @@
-# Done
-
-> NOTE: Move notes here
-
-## Configuration
-
-- Examples how to override default configuration
-
-
-
# Brain dump
@@ -148,3 +136,19 @@ Examples to demonstrate use cases.
## Test coverage
Catch up with the latest additions.
+
+
+
+# Done
+
+> NOTE: Move notes here
+
+## Configuration
+
+- Examples how to override default configuration
+
+## Evals
+
+- Simplify contracts - currently 5 (!) contracts for observations
+- Add input, output, etc. tokens default metrics
+
diff --git a/prompt.txt b/prompt.txt
new file mode 100644
index 00000000..c7112435
--- /dev/null
+++ b/prompt.txt
@@ -0,0 +1,868 @@
+Project Path: Prompt
+
+Source Tree:
+
+```
+Prompt
+├── Drivers
+│ ├── BladeDriver.php
+│ └── TwigDriver.php
+├── Contracts
+│ └── CanHandleTemplate.php
+├── Enums
+│ ├── FrontMatterFormat.php
+│ └── TemplateType.php
+├── PromptInfo.php
+├── Data
+│ └── PromptEngineConfig.php
+└── Prompt.php
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/src/Extras/Prompt/Drivers/BladeDriver.php`:
+
+```php
+config->resourcePath;
+ $cache = __DIR__ . $this->config->cachePath;
+ $extension = $this->config->extension;
+ $mode = $this->config->metadata['mode'] ?? BladeOne::MODE_AUTO;
+ $this->blade = new BladeOne($views, $cache, $mode);
+ $this->blade->setFileExtension($extension);
+ }
+
+ /**
+ * Renders a template file with the given parameters.
+ *
+ * @param string $name The name of the template file
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderFile(string $name, array $parameters = []): string {
+ return $this->blade->run($name, $parameters);
+ }
+
+ /**
+ * Renders a template from a string with the given parameters.
+ *
+ * @param string $content The template content as a string
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderString(string $content, array $parameters = []): string {
+ return $this->blade->runString($content, $parameters);
+ }
+
+ /**
+ * Gets the content of a template file.
+ *
+ * @param string $name
+ * @return string
+ */
+ public function getTemplateContent(string $name): string {
+ $templatePath = $this->blade->getTemplateFile($name);
+ if (!file_exists($templatePath)) {
+ throw new Exception("Template '$name' file does not exist: $templatePath");
+ }
+ return file_get_contents($templatePath);
+ }
+
+ /**
+ * Gets names of variables from a template content.
+ * @param string $content
+ * @return array
+ */
+ public function getVariableNames(string $content): array {
+ $variables = [];
+ preg_match_all('/{{\s*([$a-zA-Z0-9_]+)\s*}}/', $content, $matches);
+ foreach ($matches[1] as $match) {
+ $name = trim($match);
+ $name = str_starts_with($name, '$') ? substr($name, 1) : $name;
+ $variables[] = $name;
+ }
+ return array_unique($variables);
+ }
+}
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/src/Extras/Prompt/Drivers/TwigDriver.php`:
+
+```php
+config->resourcePath];
+ $extension = $this->config->extension;
+
+ $loader = new class(
+ paths: $paths,
+ fileExtension: $extension
+ ) extends FilesystemLoader {
+ private string $fileExtension;
+
+ /**
+ * Constructor for the custom FilesystemLoader.
+ *
+ * @param array $paths The paths where templates are stored
+ * @param string|null $rootPath The root path for templates
+ * @param string $fileExtension The file extension to use for templates
+ */
+ public function __construct(
+ $paths = [],
+ ?string $rootPath = null,
+ string $fileExtension = '',
+ ) {
+ parent::__construct($paths, $rootPath);
+ $this->fileExtension = $fileExtension;
+ }
+
+ /**
+ * Finds a template by its name and appends the file extension if not present.
+ *
+ * @param string $name The name of the template
+ * @param bool $throw Whether to throw an exception if the template is not found
+ * @return string The path to the template
+ */
+ protected function findTemplate(string $name, bool $throw = true): string {
+ if (pathinfo($name, PATHINFO_EXTENSION) === '') {
+ $name .= $this->fileExtension;
+ }
+ return parent::findTemplate($name, $throw);
+ }
+ };
+
+ $this->twig = new Environment(
+ loader: $loader,
+ options: ['cache' => $this->config->cachePath],
+ );
+ }
+
+ /**
+ * Renders a template file with the given parameters.
+ *
+ * @param string $name The name of the template file
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderFile(string $name, array $parameters = []): string {
+ return $this->twig->render($name, $parameters);
+ }
+
+ /**
+ * Renders a template from a string with the given parameters.
+ *
+ * @param string $content The template content as a string
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderString(string $content, array $parameters = []): string {
+ return $this->twig->createTemplate($content)->render($parameters);
+ }
+
+ /**
+ * Gets the content of a template file.
+ *
+ * @param string $name
+ * @return string
+ */
+ public function getTemplateContent(string $name): string {
+ return $this->twig->getLoader()->getSourceContext($name)->getCode();
+ }
+
+ /**
+ * Gets names of variables used in a template content.
+ *
+ * @param string $content
+ * @return array
+ * @throws \Twig\Error\SyntaxError
+ */
+ public function getVariableNames(string $content): array {
+ // make Twig Source from content string
+ $source = new Source($content, 'template');
+ // Parse the template to get its AST
+ $parsedTemplate = $this->twig->parse($this->twig->tokenize($source));
+ // Collect variables
+ $variables = $this->findVariables($parsedTemplate);
+ // Remove duplicates
+ return array_unique($variables);
+ }
+
+ // INTERNAL /////////////////////////////////////////////////
+
+ private function findVariables(Node $node): array {
+ $variables = [];
+ // Check for variable nodes and add them to the list
+ if ($node instanceof NameExpression) {
+ $variables[] = $node->getAttribute('name');
+ }
+ // Recursively search in child nodes
+ foreach ($node as $child) {
+ $childVariables = $this->findVariables($child);
+ foreach ($childVariables as $variable) {
+ $variables[] = $variable;
+ }
+ }
+ return $variables;
+ }
+}
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/src/Extras/Prompt/Contracts/CanHandleTemplate.php`:
+
+```php
+config->frontMatterTags[0] ?? '---';
+ $endTag = $this->config->frontMatterTags[1] ?? '---';
+ $format = $this->config->frontMatterFormat;
+ $this->engine = $this->makeEngine($format, $startTag, $endTag);
+
+ $document = $this->engine->parse($content);
+ $this->templateData = $document->getData();
+ $this->templateContent = $document->getContent();
+ }
+
+ public function field(string $name) : mixed {
+ return $this->templateData[$name] ?? null;
+ }
+
+ public function hasField(string $name) : bool {
+ return array_key_exists($name, $this->templateData);
+ }
+
+ public function data() : array {
+ return $this->templateData;
+ }
+
+ public function content() : string {
+ return $this->templateContent;
+ }
+
+ public function variables() : array {
+ return $this->field('variables') ?? [];
+ }
+
+ public function variableNames() : array {
+ return array_keys($this->variables());
+ }
+
+ public function hasVariables() : bool {
+ return $this->hasField('variables');
+ }
+
+ public function schema() : array {
+ return $this->field('schema') ?? [];
+ }
+
+ public function hasSchema() : bool {
+ return $this->hasField('schema');
+ }
+
+ // INTERNAL /////////////////////////////////////////////////
+
+ private function makeEngine(FrontMatterFormat $format, string $startTag, string $endTag) : FrontMatter {
+ return match($format) {
+ FrontMatterFormat::Yaml => new FrontMatter(new YamlProcessor(), $startTag, $endTag),
+ FrontMatterFormat::Json => new FrontMatter(new JsonProcessor(), $startTag, $endTag),
+ FrontMatterFormat::Toml => new FrontMatter(new TomlProcessor(), $startTag, $endTag),
+ default => throw new InvalidArgumentException("Unknown front matter format: $format->value"),
+ };
+ }
+}
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/src/Extras/Prompt/Data/PromptEngineConfig.php`:
+
+```php
+config = $config ?? PromptEngineConfig::load(
+ setting: $setting ?: Settings::get('prompt', "defaultSetting")
+ );
+ $this->driver = $driver ?? $this->makeDriver($this->config);
+ $this->templateContent = $name ? $this->load($name) : '';
+ }
+
+ public static function using(string $setting) : Prompt {
+ return new self(setting: $setting);
+ }
+
+ public static function get(string $name, string $setting = '') : Prompt {
+ return new self(name: $name, setting: $setting);
+ }
+
+ public static function text(string $name, array $variables, string $setting = '') : string {
+ return (new self(name: $name, setting: $setting))->withValues($variables)->toText();
+ }
+
+ public static function messages(string $name, array $variables, string $setting = '') : Messages {
+ return (new self(name: $name, setting: $setting))->withValues($variables)->toMessages();
+ }
+
+ public function withSetting(string $setting) : self {
+ $this->config = PromptEngineConfig::load($setting);
+ $this->driver = $this->makeDriver($this->config);
+ return $this;
+ }
+
+ public function withConfig(PromptEngineConfig $config) : self {
+ $this->config = $config;
+ $this->driver = $this->makeDriver($config);
+ return $this;
+ }
+
+ public function withDriver(CanHandleTemplate $driver) : self {
+ $this->driver = $driver;
+ return $this;
+ }
+
+ public function withTemplate(string $name) : self {
+ $this->templateContent = $this->load($name);
+ $this->promptInfo = new PromptInfo($this->templateContent, $this->config);
+ return $this;
+ }
+
+ public function withTemplateContent(string $content) : self {
+ $this->templateContent = $content;
+ $this->promptInfo = new PromptInfo($this->templateContent, $this->config);
+ return $this;
+ }
+
+ public function withValues(array $values) : self {
+ $this->variableValues = $values;
+ return $this;
+ }
+
+ public function toText() : string {
+ return $this->rendered();
+ }
+
+ public function toMessages() : Messages {
+ return $this->makeMessages($this->rendered());
+ }
+
+ public function toArray() : array {
+ return $this->toMessages()->toArray();
+ }
+
+ public function config() : PromptEngineConfig {
+ return $this->config;
+ }
+
+ public function params() : array {
+ return $this->variableValues;
+ }
+
+ public function template() : string {
+ return $this->templateContent;
+ }
+
+ public function variables() : array {
+ return $this->driver->getVariableNames($this->templateContent);
+ }
+
+ public function info() : PromptInfo {
+ return $this->promptInfo;
+ }
+
+ public function validationErrors() : array {
+ $infoVars = $this->info()->variableNames();
+ $templateVars = $this->variables();
+ $valueKeys = array_keys($this->variableValues);
+
+ $messages = [];
+ foreach($infoVars as $var) {
+ if (!in_array($var, $valueKeys)) {
+ $messages[] = "$var: variable defined in template info, but value not provided";
+ }
+ if (!in_array($var, $templateVars)) {
+ $messages[] = "$var: variable defined in template info, but not used";
+ }
+ }
+ foreach($valueKeys as $var) {
+ if (!in_array($var, $infoVars)) {
+ $messages[] = "$var: value provided, but not defined in template info";
+ }
+ if (!in_array($var, $templateVars)) {
+ $messages[] = "$var: value provided, but not used in template content";
+ }
+ }
+ foreach($templateVars as $var) {
+ if (!in_array($var, $infoVars)) {
+ $messages[] = "$var: variable used in template, but not defined in template info";
+ }
+ if (!in_array($var, $valueKeys)) {
+ $messages[] = "$var: variable used in template, but value not provided";
+ }
+ }
+ return $messages;
+ }
+
+ // INTERNAL ///////////////////////////////////////////////////
+
+ private function rendered() : string {
+ if (!isset($this->rendered)) {
+ $rendered = $this->render($this->templateContent, $this->variableValues);
+ $this->rendered = $rendered;
+ }
+ return $this->rendered;
+ }
+
+ private function makeMessages(string $text) : Messages {
+ return match(true) {
+ $this->containsXml($text) && $this->hasRoles() => $this->makeMessagesFromXml($text),
+ default => Messages::fromString($text),
+ };
+ }
+
+ private function hasRoles() : string {
+ $roleStrings = [
+ '', '', ''
+ ];
+ if (Str::contains($this->rendered(), $roleStrings)) {
+ return true;
+ }
+ return false;
+ }
+
+ private function containsXml(string $text) : bool {
+ return preg_match('/<[^>]+>/', $text) === 1;
+ }
+
+ private function makeMessagesFromXml(string $text) : Messages {
+ $messages = new Messages();
+ $xml = Xml::from($text)->wrapped('chat')->toArray();
+ // TODO: validate
+ foreach ($xml as $key => $message) {
+ $messages->appendMessage(Message::make($key, $message));
+ }
+ return $messages;
+ }
+
+ private function makeDriver(PromptEngineConfig $config) : CanHandleTemplate {
+ return match($config->templateType) {
+ TemplateType::Twig => new TwigDriver($config),
+ TemplateType::Blade => new BladeDriver($config),
+ default => throw new InvalidArgumentException("Unknown driver: $config->templateType"),
+ };
+ }
+
+ private function load(string $path) : string {
+ return $this->driver->getTemplateContent($path);
+ }
+
+ private function render(string $template, array $parameters = []) : string {
+ return $this->driver->renderString($template, $parameters);
+ }
+}
+```
+
+Project Path: prompts
+
+Source Tree:
+
+```
+prompts
+├── twig
+│ ├── summary_struct.twig
+│ ├── summary.twig
+│ ├── capital.twig
+│ └── hello.twig
+└── blade
+ ├── capital.blade.php
+ └── hello.blade.php
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/prompts/twig/summary_struct.twig`:
+
+```twig
+[Action: {{input}}] [Noun: Analyze] [Modifier: Thoroughly] [Noun: Input_Text] [Goal: Generate_Essential_Questions] [Parameter: Number=5]
+
+[Given: Essential_Questions]
+[Action: {{input}}] [Noun: Formulate_Questions] [Modifier: To Capture] [Parameter: Themes=Core Meaning, Argument, Supporting_Ideas, Author_Purpose, Implications]
+[Action: Address] [Noun: Central_Theme]
+[Action: Identify] [Noun: Key_Supporting_Ideas]
+[Action: Highlight] [Noun: Important_Facts or Evidence]
+[Action: Reveal] [Noun: Author_Purpose or Perspective]
+[Action: Explore] [Noun: Significant_Implications or Conclusions]
+
+[Action: {{input}}] [Noun: Answer_Generated_Questions] [Modifier: Thoroughly] [Parameter: Detail=High]
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/prompts/twig/summary.twig`:
+
+```twig
+{#
+---
+description: Summarize input
+params:
+ input:
+ description: Input text to summarize
+ type: string
+ depth:
+ description: Depth of summarization (number of essential questions)
+ type: int
+ default: 5
+---
+#}
+
+1) Analyze the input and generate {{ depth }} essential questions that, when answered, capture the main points and core meaning of the text.
+2) When formulating your questions:
+ - Address the central theme or argument
+ - Identify key supporting ideas
+ - Highlight important facts or evidence
+ - Reveal the author's purpose or perspective
+ - Explore any significant implications or conclusions.
+3) Answer all of your generated questions one-by-one in detail.
+
+# INPUT
+{{ input }}
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/prompts/twig/capital.twig`:
+
+```twig
+{#
+---
+description: Find country capital template for testing templates
+variables:
+ country:
+ description: country name
+ type: string
+ default: France
+schema:
+ name: capital
+ properties:
+ name:
+ description: Capital of the country
+ type: string
+ required: [name]
+---
+#}
+
+
+ You are a helpful assistant, respond to the user's questions in a concise manner.
+ Respond with JSON object, follow the format: {{ json_schema }}
+
+
+ {# examples #}
+
+
+ What is the capital of France?
+
+
+
+ {{ ['name': 'Paris'] | json_encode() }}
+
+
+ {# /examples #}
+
+
+ What is the capital of {{ country }}?
+
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/prompts/twig/hello.twig`:
+
+```twig
+{#---
+description: Hello world template for testing templates
+variables:
+ name:
+ description: Name of the person to greet
+ type: string
+ default: World
+---#}
+
+Hello, {{ name }}!
+```
+
+`/home/ddebowczyk/projects/instructor-php/prompts/blade/capital.blade.php`:
+
+```php
+{{--
+description: Find country capital template for testing templates
+variables:
+ country:
+ description: country name
+ type: string
+ default: France
+schema:
+ name: capital
+ properties:
+ name:
+ description: Capital of the country
+ type: string
+ required: [name]
+--}}
+
+
+ You are a helpful assistant, respond to the user's questions in a concise manner.
+
+
+ {{-- examples --}}
+
+
+ What is the capital of France?
+
+
+
+ {{ json_encode(['name' => 'Paris']) }}
+
+
+ {{-- /examples --}}
+
+
+ What is the capital of {{ $country }}?
+
+
+
+```
+
+`/home/ddebowczyk/projects/instructor-php/prompts/blade/hello.blade.php`:
+
+```php
+{{--
+description: Hello world template for testing templates
+variables:
+ name:
+ description: Name of the person to greet
+ type: string
+ default: World
+--}}
+
+Hello, {{ $name }}!
+```
diff --git a/prompts/blade/capital.blade.php b/prompts/blade/capital.blade.php
new file mode 100644
index 00000000..29270dde
--- /dev/null
+++ b/prompts/blade/capital.blade.php
@@ -0,0 +1,11 @@
+{{--
+---
+description: Find country capital template for testing templates
+variables:
+ country:
+ description: country name
+ type: string
+ default: France
+---
+--}}
+What is the capital of {{ $country }}?
diff --git a/prompts/blade/capital_json.blade.php b/prompts/blade/capital_json.blade.php
new file mode 100644
index 00000000..00840741
--- /dev/null
+++ b/prompts/blade/capital_json.blade.php
@@ -0,0 +1,36 @@
+{{--
+description: Find country capital template for testing templates
+variables:
+ country:
+ description: country name
+ type: string
+ default: France
+schema:
+ name: capital
+ properties:
+ name:
+ description: Capital of the country
+ type: string
+ required: [name]
+--}}
+
+
+ You are a helpful assistant, respond to the user's questions in a concise manner.
+
+
+ {{-- examples --}}
+
+
+ What is the capital of France?
+
+
+
+ {{ json_encode(['name' => 'Paris']) }}
+
+
+ {{-- /examples --}}
+
+
+ What is the capital of {{ $country }}?
+
+
diff --git a/prompts/blade/hello.blade.php b/prompts/blade/hello.blade.php
new file mode 100644
index 00000000..1468f3da
--- /dev/null
+++ b/prompts/blade/hello.blade.php
@@ -0,0 +1,10 @@
+{{--
+description: Hello world template for testing templates
+variables:
+ name:
+ description: Name of the person to greet
+ type: string
+ default: World
+--}}
+
+Hello, {{ $name }}!
\ No newline at end of file
diff --git a/prompts/twig/capital.twig b/prompts/twig/capital.twig
index 2271d80b..d771c94b 100644
--- a/prompts/twig/capital.twig
+++ b/prompts/twig/capital.twig
@@ -1,24 +1,11 @@
{#
---
description: Find country capital template for testing templates
-params:
+variables:
country:
- description: Country name
+ description: country name
type: string
default: France
---
#}
-
-{# [assistant] #}
-You are a helpful assistant, respond to the user's questions in a concise manner.
-
-{# [user] #}
-What is the capital of France?
-
-{# [assistant] #}
-{{ ['name': 'Paris'] | json_encode() }}
-
-{# [user] #}
-What is the capital of {{ country }}?
-
-{# [assistant] #}
+What is the capital of {{ country }}?
\ No newline at end of file
diff --git a/prompts/twig/capital_json.twig b/prompts/twig/capital_json.twig
new file mode 100644
index 00000000..04e27fea
--- /dev/null
+++ b/prompts/twig/capital_json.twig
@@ -0,0 +1,39 @@
+{#
+---
+description: Find country capital template for testing templates
+variables:
+ country:
+ description: country name
+ type: string
+ default: France
+schema:
+ name: capital
+ properties:
+ name:
+ description: Capital of the country
+ type: string
+ required: [name]
+---
+#}
+
+
+ You are a helpful assistant, respond to the user's questions in a concise manner.
+ Respond with JSON object, follow the format: {{ json_schema }}
+
+
+ {# examples #}
+
+
+ What is the capital of France?
+
+
+
+ {{ {'name': 'Paris'} | json_encode() }}
+
+
+ {# /examples #}
+
+
+ What is the capital of {{ country }}?
+
+
\ No newline at end of file
diff --git a/prompts/twig/hello.twig b/prompts/twig/hello.twig
index 4b6e78eb..11f6cc81 100644
--- a/prompts/twig/hello.twig
+++ b/prompts/twig/hello.twig
@@ -1,10 +1,10 @@
{#---
description: Hello world template for testing templates
-params:
+variables:
name:
description: Name of the person to greet
type: string
default: World
---#}
-Hello, {{ name }}!
+Hello, {{ name }}!
\ No newline at end of file
diff --git a/prompts/twig/summary.twig b/prompts/twig/summary.twig
new file mode 100644
index 00000000..0f4d524c
--- /dev/null
+++ b/prompts/twig/summary.twig
@@ -0,0 +1,25 @@
+{#
+---
+description: Summarize input
+params:
+ input:
+ description: Input text to summarize
+ type: string
+ depth:
+ description: Depth of summarization (number of essential questions)
+ type: int
+ default: 5
+---
+#}
+
+1) Analyze the input and generate {{ depth }} essential questions that, when answered, capture the main points and core meaning of the text.
+2) When formulating your questions:
+ - Address the central theme or argument
+ - Identify key supporting ideas
+ - Highlight important facts or evidence
+ - Reveal the author's purpose or perspective
+ - Explore any significant implications or conclusions.
+3) Answer all of your generated questions one-by-one in detail.
+
+# INPUT
+{{ input }}
diff --git a/prompts/twig/summary_struct.twig b/prompts/twig/summary_struct.twig
new file mode 100644
index 00000000..cf61deaf
--- /dev/null
+++ b/prompts/twig/summary_struct.twig
@@ -0,0 +1,11 @@
+[Action: {{input}}] [Noun: Analyze] [Modifier: Thoroughly] [Noun: Input_Text] [Goal: Generate_Essential_Questions] [Parameter: Number=5]
+
+[Given: Essential_Questions]
+[Action: {{input}}] [Noun: Formulate_Questions] [Modifier: To Capture] [Parameter: Themes=Core Meaning, Argument, Supporting_Ideas, Author_Purpose, Implications]
+[Action: Address] [Noun: Central_Theme]
+[Action: Identify] [Noun: Key_Supporting_Ideas]
+[Action: Highlight] [Noun: Important_Facts or Evidence]
+[Action: Reveal] [Noun: Author_Purpose or Perspective]
+[Action: Explore] [Noun: Significant_Implications or Conclusions]
+
+[Action: {{input}}] [Noun: Answer_Generated_Questions] [Modifier: Thoroughly] [Parameter: Detail=High]
diff --git a/src/Extras/Evals/Console/Display.php b/src/Extras/Evals/Console/Display.php
index 753a5aaf..0b1a3cc8 100644
--- a/src/Extras/Evals/Console/Display.php
+++ b/src/Extras/Evals/Console/Display.php
@@ -4,7 +4,6 @@
use Cognesy\Instructor\Extras\Evals\Execution;
use Cognesy\Instructor\Extras\Evals\Experiment;
-use Cognesy\Instructor\Extras\Evals\Observation\SelectObservations;
use Cognesy\Instructor\Utils\Cli\Color;
use Cognesy\Instructor\Utils\Cli\Console;
use Cognesy\Instructor\Utils\Debug\Debug;
@@ -20,45 +19,56 @@ public function __construct(array $options = []) {
}
public function header(Experiment $experiment) : void {
+ $id = $experiment->id();
+ $title = ' EXPERIMENT (' . Str::limit(
+ text: $id,
+ limit: 4,
+ cutMarker: '',
+ align: STR_PAD_LEFT,
+ fit: false
+ ) . ") ";
+ $startedAt = ' ' . $experiment->startedAt()->format('Y-m-d H:i:s') . ' ';
+
Console::println('');
Console::printColumns([
- [22, ' EXPERIMENT (' . Str::limit(text: $experiment->id(), limit: 4, align: STR_PAD_LEFT, fit: false) . ") ", STR_PAD_RIGHT, [Color::BG_BLUE, Color::WHITE, Color::BOLD]],
+ [22, $title, STR_PAD_RIGHT, [Color::BG_BLUE, Color::WHITE, Color::BOLD]],
[$this->flex(22, 30, -2), ' ', STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
- [30, ' ' . $experiment->startedAt()->format('Y-m-d H:i:s') . ' ', STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
+ [30, $startedAt, STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
], $this->terminalWidth, '');
Console::println('');
Console::println('');
}
public function footer(Experiment $experiment) {
+ $title = ' SUMMARY ';
+ $info = ' Time: ' . number_format($experiment->timeElapsed(), 2) . ' sec '
+ . ' ' . $experiment->usage()->toString() . ' ';
+
Console::println('');
Console::printColumns([
- [20, number_format($experiment->timeElapsed(), 2) . ' sec ', STR_PAD_LEFT, [Color::BG_BLUE, Color::WHITE, Color::BOLD]],
- [$this->flex(20, 50), ' ', STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
- [50, ' ' . $experiment->usage()->toString() . ' ', STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
+ [20, $title, STR_PAD_RIGHT, [Color::BG_BLUE, Color::WHITE, Color::BOLD]],
+ [$this->flex(20, 60), ' ', STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
+ [60, $info, STR_PAD_LEFT, [Color::BG_GRAY, Color::DARK_GRAY]],
], $this->terminalWidth, '');
Console::println('');
- Console::println('');
$this->displayObservations($experiment);
Console::println('');
}
- public function before(Execution $execution) : void {
- $id = Str::limit($execution->id(), 4, STR_PAD_LEFT);
+ public function displayExecution(Execution $execution) : void {
+ $id = Str::limit(text: $execution->id(), limit: 4, cutMarker: '', align: STR_PAD_LEFT);
$connection = $execution->get('case.connection');
$mode = $execution->get('case.mode')->value;
$streamed = $execution->get('case.isStreamed');
+ $streamLabel = $streamed ? 'stream' : 'sync';
Console::printColumns([
- [4, $id, STR_PAD_LEFT, Color::DARK_GRAY],
+ [5, $id, STR_PAD_LEFT, Color::DARK_GRAY],
[10, $connection, STR_PAD_RIGHT, Color::WHITE],
[11, $mode, STR_PAD_RIGHT, Color::YELLOW],
- [8, $streamed ? 'stream' : 'sync', STR_PAD_LEFT, $streamed ? Color::BLUE : Color::DARK_BLUE],
+ [8, $streamLabel, STR_PAD_LEFT, $streamed ? Color::BLUE : Color::DARK_BLUE],
], $this->terminalWidth);
Console::print('', [Color::GRAY, Color::BG_BLACK]);
- }
-
- public function after(Execution $execution) : void {
if ($execution->hasException()) {
$this->displayException($execution->exception());
} else {
@@ -143,7 +153,6 @@ private function exceptionToText(Exception $e, int $maxLen) : string {
private function displayObservations(Experiment $experiment)
{
- Console::println('SUMMARY:', [Color::WHITE, Color::BOLD]);
Console::printColumns([
[5, 'ID', STR_PAD_LEFT, [Color::DARK_YELLOW]],
[25, 'KEY', STR_PAD_LEFT, [Color::DARK_YELLOW]],
@@ -154,16 +163,19 @@ private function displayObservations(Experiment $experiment)
], $this->terminalWidth);
foreach ($experiment->observations() as $observation) {
- $id = Str::limit($observation->id(), 4, STR_PAD_LEFT);
+ $id = Str::limit(text: $observation->id(), limit: 4, cutMarker: '', align: STR_PAD_LEFT);
$value = $observation->value();
$unit = $observation->metadata()->get('unit', '-');
$format = $observation->metadata()->get('format', '%s');
$method = $observation->metadata()->get('aggregationMethod', '-');
- $meta = Str::limit($observation->metadata()->except('experimentId', 'unit', 'format', 'aggregationMethod')->toJson(), 60);
+ $meta = Str::limit(
+ text: $observation->metadata()->except('experimentId', 'unit', 'format', 'aggregationMethod')->toJson(),
+ limit: 60
+ );
Console::printColumns([
[5, $id, STR_PAD_LEFT, [Color::DARK_GRAY]],
- [25, $observation->key(), STR_PAD_LEFT, [Color::GRAY]],
+ [25, $observation->key(), STR_PAD_RIGHT, [Color::GRAY]],
[10, sprintf($format, $value), STR_PAD_LEFT, [Color::WHITE]],
[10, $unit, STR_PAD_RIGHT, [Color::DARK_GRAY]],
[10, $method, STR_PAD_RIGHT, [Color::DARK_GRAY]],
diff --git a/src/Extras/Evals/Dataset/Dataset.php b/src/Extras/Evals/Dataset/Dataset.php
new file mode 100644
index 00000000..694756b8
--- /dev/null
+++ b/src/Extras/Evals/Dataset/Dataset.php
@@ -0,0 +1,8 @@
+events = $events ?? new EventDispatcher();
$this->id = Uuid::uuid4();
$this->data = new DataMap();
$this->data->set('case', $case);
$this->usage = Usage::none();
}
- public function id() : string {
- return $this->id;
- }
-
- public function get(string $key) : mixed {
- return $this->data->get($key);
- }
-
- public function set(string $key, mixed $value) : self {
- $this->data->set($key, $value);
- return $this;
- }
-
- public function data() : DataMap {
- return $this->data;
- }
-
- public function withData(DataMap $data) : self {
- $this->data = $data;
- return $this;
- }
-
- public function withExecutor(CanRunExecution $executor) : self {
- $this->action = $executor;
- return $this;
- }
-
- public function withProcessors(array|object $processors) : self {
- $this->processors = match(true) {
- is_array($processors) => $processors,
- default => [$processors],
- };
- return $this;
- }
-
- public function withPostprocessors(array $processors) : self {
- $this->postprocessors = match(true) {
- is_array($processors) => $processors,
- default => [$processors],
- };
- return $this;
- }
-
- public function execute() : void {
- $this->startedAt = new DateTime();
- $time = microtime(true);
- try {
- $this->action->run($this);
- } catch(Exception $e) {
- $this->timeElapsed = microtime(true) - $time;
- $this->data()->set('output.notes', $e->getMessage());
- $this->exception = $e;
- throw $e;
- }
- $this->timeElapsed = microtime(true) - $time;
- $this->data()->set('output.notes', $this->get('response')?->content());
- $this->usage = $this->get('response')?->usage() ?? Usage::none();
- $this->observations = $this->makeObservations();
- }
-
- // HELPERS //////////////////////////////////////////////////
-
- /**
- * @return Observation[]
- */
- public function observations() : array {
- return $this->observations;
- }
-
- public function hasObservations() : bool {
- return count($this->observations) > 0;
- }
-
- // HELPERS //////////////////////////////////////////////////
-
- public function exception() : ?Exception {
- return $this->exception;
- }
-
- public function hasException() : bool {
- return $this->exception !== null;
- }
-
- public function status() : string {
- return $this->exception ? 'failed' : 'success';
- }
-
- public function startedAt() : DateTime {
- return $this->startedAt;
- }
-
- public function timeElapsed() : float {
- return $this->timeElapsed;
- }
-
- public function usage() : Usage {
- return $this->usage;
- }
-
- public function totalTps() : float {
- if ($this->timeElapsed() === 0) {
- return 0;
- }
- return $this->usage->total() / $this->timeElapsed();
- }
-
- public function outputTps() : float {
- if ($this->timeElapsed === 0) {
- return 0;
- }
- return $this->usage->output() / $this->timeElapsed();
- }
-
- /**
- * @return Observation[]
- */
- public function metrics() : array {
- return SelectObservations::from($this->observations)
- ->withTypes(['metric'])
- ->all();
- }
-
- public function hasMetrics() : bool {
- return count($this->metrics()) > 0;
- }
-
- /**
- * @return Observation[]
- */
- public function feedback() : array {
- return SelectObservations::from($this->observations)
- ->withTypes(['feedback'])
- ->all();
- }
-
- public function hasFeedback() : bool {
- return count($this->feedback()) > 0;
- }
-
- /**
- * @return Observation[]
- */
- public function summaries() : array {
- return SelectObservations::from([$this->observations])
- ->withTypes(['summary'])
- ->all();
- }
-
- public function hasSummaries() : bool {
- return count($this->summaries()) > 0;
- }
-
- private function makeObservations() : array {
- $observations = MakeObservations::for($this)
- ->withObservers([
- $this->processors,
- $this->defaultObservers,
- ])
- ->only([
- CanObserveExecution::class,
- CanGenerateObservations::class,
- ]);
-
- $summaries = MakeObservations::for($this)
- ->withObservers([
- $this->postprocessors
- ])
- ->only([
- CanObserveExecution::class,
- CanGenerateObservations::class,
- ]);
+ // PUBLIC /////////////////////////////////////////////////////////
- return array_filter(array_merge($observations, $summaries));
+ public function toArray() : array {
+ return [
+ 'id' => $this->id(),
+ 'startedAt' => $this->startedAt(),
+ 'status' => $this->status(),
+ 'data' => $this->data(),
+ 'timeElapsed' => $this->timeElapsed(),
+ 'usage' => $this->usage(),
+ 'exception' => $this->exception(),
+ ];
}
}
diff --git a/src/Extras/Evals/Experiment.php b/src/Extras/Evals/Experiment.php
index 0e4ee3f7..d30dcd95 100644
--- a/src/Extras/Evals/Experiment.php
+++ b/src/Extras/Evals/Experiment.php
@@ -1,16 +1,13 @@
events = $events ?? new EventDispatcher();
$this->id = Uuid::uuid4();
$this->display = new Display();
$this->data = new DataMap();
@@ -70,158 +73,11 @@ public function __construct(
// PUBLIC //////////////////////////////////////////////////
- public function id() : string {
- return $this->id;
- }
-
- public function startedAt() : DateTime {
- return $this->startedAt;
- }
-
- public function timeElapsed() : float {
- return $this->timeElapsed;
- }
-
- public function usage() : Usage {
- return $this->usage;
- }
-
- public function data() : DataMap {
- return $this->data;
- }
-
- /**
- * @return Observation[]
- */
- public function execute() : array {
- $this->startedAt = new DateTime();
- $this->display->header($this);
-
- // execute cases
- foreach ($this->cases as $case) {
- $this->executeCase($case);
- }
- $this->usage = $this->accumulateUsage();
- $this->timeElapsed = microtime(true) - $this->startedAt->getTimestamp();
-
- $this->observations = $this->makeObservations();
-
- $this->display->footer($this);
- if (!empty($this->exceptions)) {
- $this->display->displayExceptions($this->exceptions);
- }
-
- return $this->summaries();
- }
-
- /**
- * @return Execution[]
- */
- public function executions() : array {
- return $this->executions;
- }
-
- /**
- * @return Observation[]
- */
- public function metrics(string $name) : array {
- return SelectObservations::from($this->observations)->withTypes(['metric'])->get($name);
- }
-
- /**
- * @return Observation[]
- */
- public function summaries() : array {
- return SelectObservations::from($this->observations)->withTypes(['summary'])->all();
- }
-
- /**
- * @return Observation[]
- */
- public function feedback() : array {
- return SelectObservations::from($this->observations)->withTypes(['feedback'])->all();
- }
-
- /**
- * @return Observation[]
- */
- public function observations() : array {
- return $this->observations;
- }
-
- public function hasObservations() : bool {
- return count($this->observations) > 0;
- }
-
- /**
- * @return Observation[]
- */
- public function executionObservations() : array {
- $observations = [];
- foreach($this->executions as $execution) {
- foreach($execution->observations() as $observation) {
- $observations[] = $observation;
- }
- }
- return $observations;
- }
-
- // INTERNAL /////////////////////////////////////////////////
-
- private function executeCase(mixed $case) : void {
- $execution = $this->makeExecution($case);
- $this->display->before($execution);
- try {
- $execution->execute();
- } catch(Exception $e) {
- $this->exceptions[$execution->id()] = $execution->exception();
- }
- $this->executions[] = $execution;
- $this->display->after($execution);
- }
-
- private function makeExecution(mixed $case) : Execution {
- $caseData = match(true) {
- is_array($case) => $case,
- method_exists($case, 'toArray') => $case->toArray(),
- default => (array) $case,
- };
- return (new Execution(case: $caseData))
- ->withExecutor($this->executor)
- ->withProcessors($this->processors)
- ->withPostprocessors($this->postprocessors);
- }
-
- private function accumulateUsage() : Usage {
- $usage = new Usage();
- foreach ($this->executions as $execution) {
- $usage->accumulate($execution->usage());
- }
- return $usage;
- }
-
- private function makeObservations() : array {
- // execute observers
- $observations = MakeObservations::for($this)
- ->withObservers([
- $this->processors,
- $this->defaultProcessors,
- ])
- ->only([
- CanObserveExperiment::class,
- CanGenerateObservations::class,
- ]);
-
- // execute summarizers
- $summaries = MakeObservations::for($this)
- ->withObservers([
- $this->postprocessors,
- ])
- ->only([
- CanObserveExperiment::class,
- CanGenerateObservations::class,
- ]);
-
- return array_filter(array_merge($observations, $summaries));
+ public function toArray() : array {
+ return [
+ 'id' => $this->id,
+ 'data' => $this->data->toArray(),
+ 'executions' => array_map(fn($e) => $e->id(), $this->executions),
+ ];
}
}
diff --git a/src/Extras/Evals/Observation.php b/src/Extras/Evals/Observation.php
index 699cccf3..f229bc7a 100644
--- a/src/Extras/Evals/Observation.php
+++ b/src/Extras/Evals/Observation.php
@@ -8,6 +8,8 @@
class Observation
{
+ use Traits\Observation\HandlesAccess;
+
private readonly string $id;
private readonly DateTimeImmutable $timestamp;
private string $type;
@@ -58,16 +60,6 @@ public static function fromArray(array $data) : self {
);
}
- public function withValue(mixed $value) : self {
- $this->value = $value;
- return $this;
- }
-
- public function withMetadata(array $metadata) : self {
- $this->metadata->merge($metadata);
- return $this;
- }
-
public function toArray() : array {
return [
'id' => $this->id,
@@ -78,40 +70,4 @@ public function toArray() : array {
'metadata' => $this->metadata->toArray(),
];
}
-
- public function has(string $key) : bool {
- return $this->metadata->has($key);
- }
-
- public function get(string $key, mixed $default = null) : mixed {
- return $this->metadata->get($key, $default);
- }
-
- public function id() : string {
- return $this->id;
- }
-
- public function timestamp() : DateTimeImmutable {
- return $this->timestamp;
- }
-
- public function type() : string {
- return $this->type;
- }
-
- public function key() : string {
- return $this->key;
- }
-
- public function value() : mixed {
- return $this->value;
- }
-
- public function metadata() : DataMap {
- return $this->metadata;
- }
-
- public function toFloat() : float {
- return (float) $this->value;
- }
}
diff --git a/src/Extras/Evals/Aggregators/AggregateExperimentObservation.php b/src/Extras/Evals/Observers/Aggregate/AggregateExperimentObserver.php
similarity index 91%
rename from src/Extras/Evals/Aggregators/AggregateExperimentObservation.php
rename to src/Extras/Evals/Observers/Aggregate/AggregateExperimentObserver.php
index 006b1ed5..4b18f14e 100644
--- a/src/Extras/Evals/Aggregators/AggregateExperimentObservation.php
+++ b/src/Extras/Evals/Observers/Aggregate/AggregateExperimentObserver.php
@@ -1,6 +1,6 @@
metrics($experiment)->failureRate,
metadata: [
diff --git a/src/Extras/Evals/Observers/ExperimentLatency.php b/src/Extras/Evals/Observers/Aggregate/ExperimentLatency.php
similarity index 77%
rename from src/Extras/Evals/Observers/ExperimentLatency.php
rename to src/Extras/Evals/Observers/Aggregate/ExperimentLatency.php
index a6be89f6..6caed1d5 100644
--- a/src/Extras/Evals/Observers/ExperimentLatency.php
+++ b/src/Extras/Evals/Observers/Aggregate/ExperimentLatency.php
@@ -1,8 +1,7 @@
metricNames['precision'] ?? 'execution.precision',
value: $precision,
metadata: [
'executionId' => $execution->id(),
@@ -83,7 +84,7 @@ private function recall(Execution $execution, array $analysis) : Observation {
$recall = $analysis['true_positives'] / ($analysis['true_positives'] + $analysis['false_negatives']);
return Observation::make(
type: 'metric',
- key: 'execution.recall',
+ key: $this->metricNames['recall'] ?? 'execution.recall',
value: $recall,
metadata: [
'executionId' => $execution->id(),
@@ -102,7 +103,7 @@ private function critique(Execution $execution): array {
callback: fn(Observation $observation) => $observation->withMetadata(['executionId' => $execution->id()]),
array: $feedback->toObservations([
'executionId' => $execution->id(),
- 'key' => 'execution.field_feedback',
+ 'key' => $this->metricNames['field_feedback'] ?? 'execution.field_feedback',
])
);
}
diff --git a/src/Extras/Evals/Evaluators/Data/BooleanCorrectnessAnalysis.php b/src/Extras/Evals/Observers/Evaluate/Data/BooleanCorrectnessAnalysis.php
similarity index 88%
rename from src/Extras/Evals/Evaluators/Data/BooleanCorrectnessAnalysis.php
rename to src/Extras/Evals/Observers/Evaluate/Data/BooleanCorrectnessAnalysis.php
index 5c94c0f6..9192d233 100644
--- a/src/Extras/Evals/Evaluators/Data/BooleanCorrectnessAnalysis.php
+++ b/src/Extras/Evals/Observers/Evaluate/Data/BooleanCorrectnessAnalysis.php
@@ -1,6 +1,6 @@
id;
+ }
+
+ public function get(string $key) : mixed {
+ return $this->data->get($key);
+ }
+
+ public function set(string $key, mixed $value) : self {
+ $this->data->set($key, $value);
+ return $this;
+ }
+
+ public function data() : DataMap {
+ return $this->data;
+ }
+
+ public function withData(DataMap $data) : self {
+ $this->data = $data;
+ return $this;
+ }
+
+ public function withExecutor(CanRunExecution $executor) : self {
+ $this->action = $executor;
+ return $this;
+ }
+
+ public function withProcessors(array|object $processors) : self {
+ $this->processors = match(true) {
+ is_array($processors) => $processors,
+ default => [$processors],
+ };
+ return $this;
+ }
+
+ public function withPostprocessors(array $processors) : self {
+ $this->postprocessors = match(true) {
+ is_array($processors) => $processors,
+ default => [$processors],
+ };
+ return $this;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function observations() : array {
+ return $this->observations;
+ }
+
+ public function hasObservations() : bool {
+ return count($this->observations) > 0;
+ }
+
+ public function exception() : ?Exception {
+ return $this->exception;
+ }
+
+ public function hasException() : bool {
+ return $this->exception !== null;
+ }
+
+ public function status() : string {
+ return $this->exception ? 'failed' : 'success';
+ }
+
+ public function startedAt() : DateTime {
+ return $this->startedAt;
+ }
+
+ public function timeElapsed() : float {
+ return $this->timeElapsed;
+ }
+
+ public function usage() : Usage {
+ return $this->usage;
+ }
+
+ public function totalTps() : float {
+ if ($this->timeElapsed() === 0) {
+ return 0;
+ }
+ return $this->usage->total() / $this->timeElapsed();
+ }
+
+ public function outputTps() : float {
+ if ($this->timeElapsed === 0) {
+ return 0;
+ }
+ return $this->usage->output() / $this->timeElapsed();
+ }
+
+ public function hasMetrics() : bool {
+ return count($this->metrics()) > 0;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function metrics() : array {
+ return SelectObservations::from($this->observations)
+ ->withTypes(['metric'])
+ ->all();
+ }
+
+ public function hasFeedback() : bool {
+ return count($this->feedback()) > 0;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function feedback() : array {
+ return SelectObservations::from($this->observations)
+ ->withTypes(['feedback'])
+ ->all();
+ }
+
+ public function hasSummaries() : bool {
+ return count($this->summaries()) > 0;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function summaries() : array {
+ return SelectObservations::from([$this->observations])
+ ->withTypes(['summary'])
+ ->all();
+ }
+}
\ No newline at end of file
diff --git a/src/Extras/Evals/Traits/Execution/HandlesExecution.php b/src/Extras/Evals/Traits/Execution/HandlesExecution.php
new file mode 100644
index 00000000..66d6a517
--- /dev/null
+++ b/src/Extras/Evals/Traits/Execution/HandlesExecution.php
@@ -0,0 +1,61 @@
+startedAt = new DateTime();
+ $time = microtime(true);
+ try {
+ $this->action->run($this);
+ $this->events->dispatch(new ExecutionDone($this->toArray()));
+ } catch(Exception $e) {
+ $this->timeElapsed = microtime(true) - $time;
+ $this->data()->set('output.notes', $e->getMessage());
+ $this->exception = $e;
+ $this->events->dispatch(new ExecutionFailed($this->toArray()));
+ throw $e;
+ }
+ $this->timeElapsed = microtime(true) - $time;
+ $this->data()->set('output.notes', $this->get('response')?->content());
+ $this->usage = $this->get('response')?->usage() ?? Usage::none();
+ $this->observations = $this->makeObservations();
+ $this->events->dispatch(new ExecutionProcessed($this->toArray()));
+ }
+
+ // INTERNAL //////////////////////////////////////////////////
+
+ private function makeObservations() : array {
+ $observations = MakeObservations::for($this)
+ ->withObservers([
+ $this->processors,
+ $this->defaultObservers,
+ ])
+ ->only([
+ CanObserveExecution::class,
+ CanGenerateObservations::class,
+ ]);
+
+ $summaries = MakeObservations::for($this)
+ ->withObservers([
+ $this->postprocessors
+ ])
+ ->only([
+ CanObserveExecution::class,
+ CanGenerateObservations::class,
+ ]);
+
+ return array_filter(array_merge($observations, $summaries));
+ }
+}
\ No newline at end of file
diff --git a/src/Extras/Evals/Traits/Experiment/HandlesAccess.php b/src/Extras/Evals/Traits/Experiment/HandlesAccess.php
new file mode 100644
index 00000000..0261a841
--- /dev/null
+++ b/src/Extras/Evals/Traits/Experiment/HandlesAccess.php
@@ -0,0 +1,86 @@
+id;
+ }
+
+ public function startedAt() : DateTime {
+ return $this->startedAt;
+ }
+
+ public function timeElapsed() : float {
+ return $this->timeElapsed;
+ }
+
+ public function usage() : Usage {
+ return $this->usage;
+ }
+
+ public function data() : DataMap {
+ return $this->data;
+ }
+
+ /**
+ * @return Execution[]
+ */
+ public function executions() : array {
+ return $this->executions;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function metrics(string $name) : array {
+ return SelectObservations::from($this->observations)->withTypes(['metric'])->get($name);
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function summaries() : array {
+ return SelectObservations::from($this->observations)->withTypes(['summary'])->all();
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function feedback() : array {
+ return SelectObservations::from($this->observations)->withTypes(['feedback'])->all();
+ }
+
+ public function hasObservations() : bool {
+ return count($this->observations) > 0;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function observations() : array {
+ return $this->observations;
+ }
+
+ /**
+ * @return Observation[]
+ */
+ public function executionObservations() : array {
+ $observations = [];
+ foreach($this->executions as $execution) {
+ foreach($execution->observations() as $observation) {
+ $observations[] = $observation;
+ }
+ }
+ return $observations;
+ }
+
+}
\ No newline at end of file
diff --git a/src/Extras/Evals/Traits/Experiment/HandlesExecution.php b/src/Extras/Evals/Traits/Experiment/HandlesExecution.php
new file mode 100644
index 00000000..99486dff
--- /dev/null
+++ b/src/Extras/Evals/Traits/Experiment/HandlesExecution.php
@@ -0,0 +1,91 @@
+startedAt = new DateTime();
+ $this->events->dispatch(new ExperimentStarted($this->toArray()));
+ $this->display->header($this);
+
+ // execute cases
+ foreach ($this->cases as $case) {
+ $execution = $this->executeCase($case);
+ $this->display->displayExecution($execution);
+ }
+ $this->usage = $this->accumulateUsage();
+ $this->timeElapsed = microtime(true) - $this->startedAt->getTimestamp();
+
+ $this->observations = $this->makeObservations();
+
+ $this->display->footer($this);
+ if (!empty($this->exceptions)) {
+ $this->display->displayExceptions($this->exceptions);
+ }
+
+ $this->events->dispatch(new ExperimentDone($this->toArray()));
+ return $this->summaries();
+ }
+
+ // INTERNAL /////////////////////////////////////////////////
+
+ private function executeCase(mixed $case) : Execution {
+ $execution = $this->makeExecution($case);
+ try {
+ $execution->execute();
+ } catch(Exception $e) {
+ $this->exceptions[$execution->id()] = $execution->exception();
+ }
+ $this->executions[] = $execution;
+ return $execution;
+ }
+
+ private function makeExecution(mixed $case) : Execution {
+ $caseData = match(true) {
+ is_array($case) => $case,
+ method_exists($case, 'toArray') => $case->toArray(),
+ default => (array) $case,
+ };
+ return (new Execution(case: $caseData))
+ ->withExecutor($this->executor)
+ ->withProcessors($this->processors)
+ ->withPostprocessors($this->postprocessors);
+ }
+
+ private function accumulateUsage() : Usage {
+ $usage = new Usage();
+ foreach ($this->executions as $execution) {
+ $usage->accumulate($execution->usage());
+ }
+ return $usage;
+ }
+
+ private function makeObservations() : array {
+ // execute observers
+ $observations = MakeObservations::for($this)
+ ->withObservers([$this->processors, $this->defaultProcessors])
+ ->only([CanObserveExperiment::class, CanGenerateObservations::class]);
+
+ // execute summarizers
+ $summaries = MakeObservations::for($this)
+ ->withObservers([$this->postprocessors])
+ ->only([CanObserveExperiment::class, CanGenerateObservations::class]);
+
+ return array_filter(array_merge($observations, $summaries));
+ }
+}
\ No newline at end of file
diff --git a/src/Extras/Evals/Traits/Observation/HandlesAccess.php b/src/Extras/Evals/Traits/Observation/HandlesAccess.php
new file mode 100644
index 00000000..b65e1005
--- /dev/null
+++ b/src/Extras/Evals/Traits/Observation/HandlesAccess.php
@@ -0,0 +1,55 @@
+value = $value;
+ return $this;
+ }
+
+ public function withMetadata(array $metadata) : self {
+ $this->metadata->merge($metadata);
+ return $this;
+ }
+
+ public function has(string $key) : bool {
+ return $this->metadata->has($key);
+ }
+
+ public function get(string $key, mixed $default = null) : mixed {
+ return $this->metadata->get($key, $default);
+ }
+
+ public function id() : string {
+ return $this->id;
+ }
+
+ public function timestamp() : DateTimeImmutable {
+ return $this->timestamp;
+ }
+
+ public function type() : string {
+ return $this->type;
+ }
+
+ public function key() : string {
+ return $this->key;
+ }
+
+ public function value() : mixed {
+ return $this->value;
+ }
+
+ public function metadata() : DataMap {
+ return $this->metadata;
+ }
+
+ public function toFloat() : float {
+ return (float) $this->value;
+ }
+}
\ No newline at end of file
diff --git a/src/Extras/Prompt/Contracts/CanHandleTemplate.php b/src/Extras/Prompt/Contracts/CanHandleTemplate.php
new file mode 100644
index 00000000..269541fe
--- /dev/null
+++ b/src/Extras/Prompt/Contracts/CanHandleTemplate.php
@@ -0,0 +1,45 @@
+config->resourcePath;
+ $cache = __DIR__ . $this->config->cachePath;
+ $extension = $this->config->extension;
+ $mode = $this->config->metadata['mode'] ?? BladeOne::MODE_AUTO;
+ $this->blade = new BladeOne($views, $cache, $mode);
+ $this->blade->setFileExtension($extension);
+ }
+
+ /**
+ * Renders a template file with the given parameters.
+ *
+ * @param string $name The name of the template file
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderFile(string $name, array $parameters = []): string {
+ return $this->blade->run($name, $parameters);
+ }
+
+ /**
+ * Renders a template from a string with the given parameters.
+ *
+ * @param string $content The template content as a string
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderString(string $content, array $parameters = []): string {
+ return $this->blade->runString($content, $parameters);
+ }
+
+ /**
+ * Gets the content of a template file.
+ *
+ * @param string $name
+ * @return string
+ */
+ public function getTemplateContent(string $name): string {
+ $templatePath = $this->blade->getTemplateFile($name);
+ if (!file_exists($templatePath)) {
+ throw new Exception("Template '$name' file does not exist: $templatePath");
+ }
+ return file_get_contents($templatePath);
+ }
+
+ /**
+ * Gets names of variables from a template content.
+ * @param string $content
+ * @return array
+ */
+ public function getVariableNames(string $content): array {
+ $variables = [];
+ preg_match_all('/{{\s*([$a-zA-Z0-9_]+)\s*}}/', $content, $matches);
+ foreach ($matches[1] as $match) {
+ $name = trim($match);
+ $name = str_starts_with($name, '$') ? substr($name, 1) : $name;
+ $variables[] = $name;
+ }
+ return array_unique($variables);
+ }
+}
diff --git a/src/Extras/Prompt/Drivers/TwigDriver.php b/src/Extras/Prompt/Drivers/TwigDriver.php
index 697bf3c7..b7098c13 100644
--- a/src/Extras/Prompt/Drivers/TwigDriver.php
+++ b/src/Extras/Prompt/Drivers/TwigDriver.php
@@ -2,7 +2,143 @@
namespace Cognesy\Instructor\Extras\Prompt\Drivers;
-class TwigDriver
+use Cognesy\Instructor\Extras\Prompt\Contracts\CanHandleTemplate;
+use Cognesy\Instructor\Extras\Prompt\Data\PromptEngineConfig;
+use Cognesy\Instructor\Utils\Arrays;
+use Twig\Environment;
+use Twig\Loader\FilesystemLoader;
+use Twig\Node\Expression\NameExpression;
+use Twig\Node\Node;
+use Twig\Source;
+
+/**
+ * Class TwigDriver
+ *
+ * Handles the rendering of Twig templates with custom file extensions and front matter support.
+ */
+class TwigDriver implements CanHandleTemplate
{
+ private Environment $twig;
+
+ /**
+ * TwigDriver constructor.
+ *
+ * @param PromptEngineConfig $config The configuration for the prompt engine
+ */
+ public function __construct(
+ private PromptEngineConfig $config,
+ ) {
+ $paths = [__DIR__ . $this->config->resourcePath];
+ $extension = $this->config->extension;
+
+ $loader = new class(
+ paths: $paths,
+ fileExtension: $extension
+ ) extends FilesystemLoader {
+ private string $fileExtension;
+
+ /**
+ * Constructor for the custom FilesystemLoader.
+ *
+ * @param array $paths The paths where templates are stored
+ * @param string|null $rootPath The root path for templates
+ * @param string $fileExtension The file extension to use for templates
+ */
+ public function __construct(
+ $paths = [],
+ ?string $rootPath = null,
+ string $fileExtension = '',
+ ) {
+ parent::__construct($paths, $rootPath);
+ $this->fileExtension = $fileExtension;
+ }
+
+ /**
+ * Finds a template by its name and appends the file extension if not present.
+ *
+ * @param string $name The name of the template
+ * @param bool $throw Whether to throw an exception if the template is not found
+ * @return string The path to the template
+ */
+ protected function findTemplate(string $name, bool $throw = true): string {
+ if (pathinfo($name, PATHINFO_EXTENSION) === '') {
+ $name .= $this->fileExtension;
+ }
+ return parent::findTemplate($name, $throw);
+ }
+ };
+
+ $this->twig = new Environment(
+ loader: $loader,
+ options: ['cache' => $this->config->cachePath],
+ );
+ }
+
+ /**
+ * Renders a template file with the given parameters.
+ *
+ * @param string $name The name of the template file
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderFile(string $name, array $parameters = []): string {
+ return $this->twig->render($name, $parameters);
+ }
+
+ /**
+ * Renders a template from a string with the given parameters.
+ *
+ * @param string $content The template content as a string
+ * @param array $parameters The parameters to pass to the template
+ * @return string The rendered template
+ */
+ public function renderString(string $content, array $parameters = []): string {
+ return $this->twig->createTemplate($content)->render($parameters);
+ }
+
+ /**
+ * Gets the content of a template file.
+ *
+ * @param string $name
+ * @return string
+ */
+ public function getTemplateContent(string $name): string {
+ return $this->twig->getLoader()->getSourceContext($name)->getCode();
+ }
+
+ /**
+ * Gets names of variables used in a template content.
+ *
+ * @param string $content
+ * @return array
+ * @throws \Twig\Error\SyntaxError
+ */
+ public function getVariableNames(string $content): array {
+ // make Twig Source from content string
+ $source = new Source($content, 'template');
+ // Parse the template to get its AST
+ $parsedTemplate = $this->twig->parse($this->twig->tokenize($source));
+ // Collect variables
+ $variables = $this->findVariables($parsedTemplate);
+ // Remove duplicates
+ return array_unique($variables);
+ }
+
+ // INTERNAL /////////////////////////////////////////////////
-}
\ No newline at end of file
+ private function findVariables(Node $node): array {
+ $variables = [];
+ // Check for variable nodes and add them to the list
+ if ($node instanceof NameExpression) {
+ $variables[] = $node->getAttribute('name');
+ }
+ // Recursively search in child nodes
+ foreach ($node as $child) {
+ $childVariables = $this->findVariables($child);
+ foreach ($childVariables as $variable) {
+ $variables[] = $variable;
+ }
+ }
+ return $variables;
+ }
+}
diff --git a/src/Extras/Prompt/Enums/FrontMatterFormat.php b/src/Extras/Prompt/Enums/FrontMatterFormat.php
new file mode 100644
index 00000000..f1b92e96
--- /dev/null
+++ b/src/Extras/Prompt/Enums/FrontMatterFormat.php
@@ -0,0 +1,11 @@
+config = $config ?? PromptEngineConfig::load(
+ setting: $setting ?: Settings::get('prompt', "defaultSetting")
+ );
+ $this->driver = $driver ?? $this->makeDriver($this->config);
+ $this->templateContent = $name ? $this->load($name) : '';
+ }
+
+ public static function using(string $setting) : Prompt {
+ return new self(setting: $setting);
+ }
+
+ public static function get(string $name, string $setting = '') : Prompt {
+ return new self(name: $name, setting: $setting);
+ }
+
+ public static function text(string $name, array $variables, string $setting = '') : string {
+ return (new self(name: $name, setting: $setting))->withValues($variables)->toText();
+ }
+
+ public static function messages(string $name, array $variables, string $setting = '') : Messages {
+ return (new self(name: $name, setting: $setting))->withValues($variables)->toMessages();
+ }
+
+ public function withSetting(string $setting) : self {
+ $this->config = PromptEngineConfig::load($setting);
+ $this->driver = $this->makeDriver($this->config);
+ return $this;
+ }
+
+ public function withConfig(PromptEngineConfig $config) : self {
+ $this->config = $config;
+ $this->driver = $this->makeDriver($config);
+ return $this;
+ }
+
+ public function withDriver(CanHandleTemplate $driver) : self {
+ $this->driver = $driver;
+ return $this;
+ }
+
+ public function withTemplate(string $name) : self {
+ $this->templateContent = $this->load($name);
+ $this->promptInfo = new PromptInfo($this->templateContent, $this->config);
+ return $this;
+ }
+
+ public function withTemplateContent(string $content) : self {
+ $this->templateContent = $content;
+ $this->promptInfo = new PromptInfo($this->templateContent, $this->config);
+ return $this;
+ }
+
+ public function withValues(array $values) : self {
+ $this->variableValues = $values;
+ return $this;
+ }
+
+ public function toText() : string {
+ return $this->rendered();
+ }
+
+ public function toMessages() : Messages {
+ return $this->makeMessages($this->rendered());
+ }
+
+ public function toArray() : array {
+ return $this->toMessages()->toArray();
+ }
+
+ public function config() : PromptEngineConfig {
+ return $this->config;
+ }
+
+ public function params() : array {
+ return $this->variableValues;
+ }
+
+ public function template() : string {
+ return $this->templateContent;
+ }
+
+ public function variables() : array {
+ return $this->driver->getVariableNames($this->templateContent);
+ }
+
+ public function info() : PromptInfo {
+ return $this->promptInfo;
+ }
+
+ public function validationErrors() : array {
+ $infoVars = $this->info()->variableNames();
+ $templateVars = $this->variables();
+ $valueKeys = array_keys($this->variableValues);
+
+ $messages = [];
+ foreach($infoVars as $var) {
+ if (!in_array($var, $valueKeys)) {
+ $messages[] = "$var: variable defined in template info, but value not provided";
+ }
+ if (!in_array($var, $templateVars)) {
+ $messages[] = "$var: variable defined in template info, but not used";
+ }
+ }
+ foreach($valueKeys as $var) {
+ if (!in_array($var, $infoVars)) {
+ $messages[] = "$var: value provided, but not defined in template info";
+ }
+ if (!in_array($var, $templateVars)) {
+ $messages[] = "$var: value provided, but not used in template content";
+ }
+ }
+ foreach($templateVars as $var) {
+ if (!in_array($var, $infoVars)) {
+ $messages[] = "$var: variable used in template, but not defined in template info";
+ }
+ if (!in_array($var, $valueKeys)) {
+ $messages[] = "$var: variable used in template, but value not provided";
+ }
+ }
+ return $messages;
+ }
+
+ // INTERNAL ///////////////////////////////////////////////////
+
+ private function rendered() : string {
+ if (!isset($this->rendered)) {
+ $rendered = $this->render($this->templateContent, $this->variableValues);
+ $this->rendered = $rendered;
+ }
+ return $this->rendered;
+ }
+
+ private function makeMessages(string $text) : Messages {
+ return match(true) {
+ $this->containsXml($text) && $this->hasRoles() => $this->makeMessagesFromXml($text),
+ default => Messages::fromString($text),
+ };
+ }
+
+ private function hasRoles() : string {
+ $roleStrings = [
+ '', '', ''
+ ];
+ if (Str::contains($this->rendered(), $roleStrings)) {
+ return true;
+ }
+ return false;
+ }
+
+ private function containsXml(string $text) : bool {
+ return preg_match('/<[^>]+>/', $text) === 1;
+ }
+
+ private function makeMessagesFromXml(string $text) : Messages {
+ $messages = new Messages();
+ $xml = Xml::from($text)->wrapped('chat')->toArray();
+ // TODO: validate
+ foreach ($xml as $key => $message) {
+ $messages->appendMessage(Message::make($key, $message));
+ }
+ return $messages;
+ }
+
+ private function makeDriver(PromptEngineConfig $config) : CanHandleTemplate {
+ return match($config->templateType) {
+ TemplateType::Twig => new TwigDriver($config),
+ TemplateType::Blade => new BladeDriver($config),
+ default => throw new InvalidArgumentException("Unknown driver: $config->templateType"),
+ };
+ }
+
+ private function load(string $path) : string {
+ return $this->driver->getTemplateContent($path);
+ }
+ private function render(string $template, array $parameters = []) : string {
+ return $this->driver->renderString($template, $parameters);
+ }
}
\ No newline at end of file
diff --git a/src/Extras/Prompt/PromptInfo.php b/src/Extras/Prompt/PromptInfo.php
new file mode 100644
index 00000000..a97ebcab
--- /dev/null
+++ b/src/Extras/Prompt/PromptInfo.php
@@ -0,0 +1,79 @@
+config->frontMatterTags[0] ?? '---';
+ $endTag = $this->config->frontMatterTags[1] ?? '---';
+ $format = $this->config->frontMatterFormat;
+ $this->engine = $this->makeEngine($format, $startTag, $endTag);
+
+ $document = $this->engine->parse($content);
+ $this->templateData = $document->getData();
+ $this->templateContent = $document->getContent();
+ }
+
+ public function field(string $name) : mixed {
+ return $this->templateData[$name] ?? null;
+ }
+
+ public function hasField(string $name) : bool {
+ return array_key_exists($name, $this->templateData);
+ }
+
+ public function data() : array {
+ return $this->templateData;
+ }
+
+ public function content() : string {
+ return $this->templateContent;
+ }
+
+ public function variables() : array {
+ return $this->field('variables') ?? [];
+ }
+
+ public function variableNames() : array {
+ return array_keys($this->variables());
+ }
+
+ public function hasVariables() : bool {
+ return $this->hasField('variables');
+ }
+
+ public function schema() : array {
+ return $this->field('schema') ?? [];
+ }
+
+ public function hasSchema() : bool {
+ return $this->hasField('schema');
+ }
+
+ // INTERNAL /////////////////////////////////////////////////
+
+ private function makeEngine(FrontMatterFormat $format, string $startTag, string $endTag) : FrontMatter {
+ return match($format) {
+ FrontMatterFormat::Yaml => new FrontMatter(new YamlProcessor(), $startTag, $endTag),
+ FrontMatterFormat::Json => new FrontMatter(new JsonProcessor(), $startTag, $endTag),
+ FrontMatterFormat::Toml => new FrontMatter(new TomlProcessor(), $startTag, $endTag),
+ default => throw new InvalidArgumentException("Unknown front matter format: $format->value"),
+ };
+ }
+}
diff --git a/src/Features/LLM/Inference.php b/src/Features/LLM/Inference.php
index 7c0757b2..290b6cd9 100644
--- a/src/Features/LLM/Inference.php
+++ b/src/Features/LLM/Inference.php
@@ -55,8 +55,8 @@ public function __construct(
EventDispatcher $events = null,
) {
$this->events = $events ?? new EventDispatcher();
- $this->config = $config ?? LLMConfig::load(connection: $connection
- ?: Settings::get('llm', "defaultConnection")
+ $this->config = $config ?? LLMConfig::load(
+ connection: $connection ?: Settings::get('llm', "defaultConnection")
);
$this->httpClient = $httpClient ?? HttpClient::make($this->config->httpClient);
$this->driver = $driver ?? $this->makeDriver($this->config, $this->httpClient);
diff --git a/src/Utils/Cli/Console.php b/src/Utils/Cli/Console.php
index 1060da7b..499a78ab 100644
--- a/src/Utils/Cli/Console.php
+++ b/src/Utils/Cli/Console.php
@@ -44,7 +44,7 @@ public static function columns(array $columns, int $maxWidth, string $divider =
$message .= self::color($color);
}
$message .= self::toColumn(
- chars: $row[0],
+ width: $row[0],
text: $row[1],
align: $row[2]??STR_PAD_RIGHT
);
@@ -55,17 +55,14 @@ public static function columns(array $columns, int $maxWidth, string $divider =
return $message;
}
- static private function toColumn(int $chars, mixed $text, int $align, string|array $color = ''): string {
-// $short = ($align === STR_PAD_LEFT)
-// ? substr($text, -$chars)
-// : substr($text, 0, $chars);
-// if ($text !== $short) {
-// $short = ($align === STR_PAD_LEFT)
-// ? '…'.substr($short,1)
-// : substr($short, 0, -1).'…';
-// }
- $short = Str::limit($text, $chars, '…', $align);
- return self::color($color, str_pad($short, $chars, ' ', $align));
+ static private function toColumn(
+ int $width,
+ mixed $text,
+ int $align,
+ string|array $color = ''
+ ): string {
+ $short = Str::limit(text: $text, limit: $width, align: $align);
+ return self::color($color, str_pad($short, $width, ' ', $align));
}
static private function color(string|array $color, string $output = '') : string {
diff --git a/src/Utils/Git/Branch.php b/src/Utils/Git/Branch.php
new file mode 100644
index 00000000..fcb8f5e4
--- /dev/null
+++ b/src/Utils/Git/Branch.php
@@ -0,0 +1,45 @@
+branchName = $branchName;
+ $this->gitService = $gitService;
+ }
+
+ public function create(): self {
+ $this->gitService->runCommand(sprintf('checkout -b %s', escapeshellarg($this->branchName)));
+ return $this;
+ }
+
+ public function delete(bool $force = false): self {
+ $command = $force ? 'branch -D' : 'branch -d';
+ $this->gitService->runCommand(sprintf('%s %s', $command, escapeshellarg($this->branchName)));
+ return $this;
+ }
+
+ public function rebase(): self {
+ $this->gitService->runCommand(sprintf('rebase %s', escapeshellarg($this->branchName)));
+ return $this;
+ }
+
+ public function merge(): self {
+ $this->gitService->runCommand(sprintf('merge %s', escapeshellarg($this->branchName)));
+ return $this;
+ }
+
+ public function push(string $remote = 'origin'): self {
+ $this->gitService->runCommand(sprintf('push %s %s', escapeshellarg($remote), escapeshellarg($this->branchName)));
+ return $this;
+ }
+
+ public function pull(string $remote = 'origin'): self {
+ $this->gitService->runCommand(sprintf('pull %s %s', escapeshellarg($remote), escapeshellarg($this->branchName)));
+ return $this;
+ }
+}
\ No newline at end of file
diff --git a/src/Utils/Git/Branches.php b/src/Utils/Git/Branches.php
index aba6aa8d..7038b207 100644
--- a/src/Utils/Git/Branches.php
+++ b/src/Utils/Git/Branches.php
@@ -6,32 +6,16 @@ class Branches
{
protected GitService $gitService;
- public function __construct(GitService $gitService)
- {
+ public function __construct(GitService $gitService) {
$this->gitService = $gitService;
}
- public function current(): string
- {
- return $this->gitService->runCommand('rev-parse --abbrev-ref HEAD');
- }
-
- public function all(): array
- {
+ public function all(): array {
$branches = $this->gitService->runCommand('branch');
return array_map('trim', explode("\n", $branches));
}
- public function create(string $branchName): self
- {
- $this->gitService->runCommand(sprintf('checkout -b %s', escapeshellarg($branchName)));
- return $this;
- }
-
- public function delete(string $branchName, bool $force = false): self
- {
- $command = $force ? 'branch -D' : 'branch -d';
- $this->gitService->runCommand(sprintf('%s %s', $command, escapeshellarg($branchName)));
- return $this;
+ public function current(): string {
+ return $this->gitService->runCommand('rev-parse --abbrev-ref HEAD');
}
}
diff --git a/src/Utils/Git/Changes.php b/src/Utils/Git/Changes.php
new file mode 100644
index 00000000..6ef4c3a4
--- /dev/null
+++ b/src/Utils/Git/Changes.php
@@ -0,0 +1,62 @@
+gitService = $gitService;
+ }
+
+ public function last(int $days): self
+ {
+ $since = (new DateTime())->modify(sprintf('-%d days', $days))->format('Y-m-d');
+ $command = sprintf("log --since=%s --pretty=format:%%H", escapeshellarg($since));
+ $this->commits = array_filter(array_map('trim', explode("\n", $this->gitService->runCommand($command))));
+ return $this;
+ }
+
+ public function files(): array
+ {
+ $files = [];
+ foreach ($this->commits as $commit) {
+ $command = sprintf('diff-tree --no-commit-id --name-only -r %s', escapeshellarg($commit));
+ $changedFiles = array_filter(array_map('trim', explode("\n", $this->gitService->runCommand($command))));
+ foreach($changedFiles as $file) {
+ $files[] = $file;
+ }
+ }
+ return array_unique($files);
+ }
+
+ public function content(): array
+ {
+ $contentChanges = [];
+ foreach ($this->files() as $file) {
+ $command = sprintf('show %s:%s', escapeshellarg($this->commits[0]), escapeshellarg($file));
+ $content = $this->gitService->runCommand($command);
+ $contentChanges[] = ['file' => $file, 'content' => $content];
+ }
+ return $contentChanges;
+ }
+
+ public function diffs(): array
+ {
+ $diffs = [];
+ foreach ($this->commits as $commit) {
+ foreach ($this->files() as $file) {
+ $command = sprintf('diff %s %s -- %s', escapeshellarg($commit . "^"), escapeshellarg($commit), escapeshellarg($file));
+ $diff = $this->gitService->runCommand($command);
+ $diffs[] = ['file' => $file, 'diff' => $diff];
+ }
+ }
+ return $diffs;
+ }
+}
diff --git a/src/Utils/Git/Commit.php b/src/Utils/Git/Commit.php
index 5c81c25e..fcbd85d0 100644
--- a/src/Utils/Git/Commit.php
+++ b/src/Utils/Git/Commit.php
@@ -2,34 +2,50 @@
namespace Cognesy\Instructor\Utils\Git;
+use DateTime;
+
class Commit
{
- protected string $hash;
protected GitService $gitService;
+ protected string $hash;
+ protected string $message;
+ protected string $author;
+ protected DateTime $date;
+ protected string $diff;
- public function __construct(string $hash, GitService $gitService)
- {
- $this->hash = $hash;
+ public function __construct(
+ string $hash,
+ GitService $gitService
+ ) {
$this->gitService = $gitService;
+ $this->hash = $hash;
}
- public function hash(): string
- {
+ public function hash(): string {
return $this->hash;
}
- public function author(): string
- {
- return $this->gitService->runCommand(sprintf('log -1 --pretty=format:%%an %%ae %s', escapeshellarg($this->hash)));
+ public function author(): string {
+ if (!isset($this->author)) {
+ $author = $this->gitService->runCommand(sprintf('log -1 --pretty=format:%%an %%ae %s', escapeshellarg($this->hash)));
+ $this->author = $author;
+ }
+ return $this->author;
}
- public function message(): string
- {
- return $this->gitService->runCommand(sprintf('log -1 --pretty=format:%%B %s', escapeshellarg($this->hash)));
+ public function message(): string {
+ if (!isset($this->message)) {
+ $message = $this->gitService->runCommand(sprintf('log -1 --pretty=format:%%B %s', escapeshellarg($this->hash)));
+ $this->message = $message;
+ }
+ return $this->message;
}
- public function date(): string
- {
- return $this->gitService->runCommand(sprintf('log -1 --pretty=format:%%ad %s', escapeshellarg($this->hash)));
+ public function date(): DateTime {
+ if (!isset($this->date)) {
+ $date = $this->gitService->runCommand(sprintf('log -1 --pretty=format:%%ad %s', escapeshellarg($this->hash)));
+ $this->date = new DateTime($date);
+ }
+ return $this->date;
}
}
diff --git a/src/Utils/Git/Diff.php b/src/Utils/Git/Diff.php
index d9cadae1..85453c9c 100644
--- a/src/Utils/Git/Diff.php
+++ b/src/Utils/Git/Diff.php
@@ -6,18 +6,15 @@ class Diff
{
protected GitService $gitService;
- public function __construct(GitService $gitService)
- {
+ public function __construct(GitService $gitService) {
$this->gitService = $gitService;
}
- public function against(string $target = 'HEAD'): string
- {
+ public function against(string $target = 'HEAD'): string {
return $this->gitService->runCommand(sprintf('diff %s', escapeshellarg($target)));
}
- public function between(string $commitA, string $commitB): string
- {
+ public function between(string $commitA, string $commitB): string {
return $this->gitService->runCommand(sprintf('diff %s %s', escapeshellarg($commitA), escapeshellarg($commitB)));
}
}
diff --git a/src/Utils/Git/File.php b/src/Utils/Git/File.php
index 372de3d7..5b74bae1 100644
--- a/src/Utils/Git/File.php
+++ b/src/Utils/Git/File.php
@@ -5,32 +5,43 @@
class File
{
protected GitService $gitService;
+ protected string $path;
- public function __construct(GitService $gitService)
- {
+ public function __construct(
+ string $path,
+ GitService $gitService,
+ ) {
+ $this->path = $path;
$this->gitService = $gitService;
}
- public function log(string $path, int $number = 10): array
- {
- $log = $this->gitService->runCommand(sprintf('log -n %d --pretty=format:%%H -- %s', $number, escapeshellarg($path)));
+ /**
+ * Retrieves a list of commit hashes for a specified path in the git repository.
+ *
+ * @param string $path The file path to get the log for.
+ * @param int $number (optional) The number of commits to retrieve. Default is 10.
+ * @return array An array of Commit objects corresponding to the retrieved hashes.
+ */
+ public function log(int $number = 10): array {
+ $log = $this->gitService->runCommand(sprintf('log -n %d --pretty=format:%%H -- %s', $number, escapeshellarg($this->path)));
$hashes = array_filter(array_map('trim', explode("\n", $log)));
return array_map(fn($hash) => new Commit($hash, $this->gitService), $hashes);
}
- public function diff(string $path): string
- {
- return $this->gitService->runCommand(sprintf('diff HEAD -- %s', escapeshellarg($path)));
- }
-
- public function versions(string $path, int $number = 10): array
- {
- $log = $this->gitService->runCommand(sprintf('log -n %d --pretty=format:%%H -- %s', $number, escapeshellarg($path)));
+ /**
+ * Retrieves a list of file versions from a git repository.
+ *
+ * @param string $path The file path for which to retrieve versions.
+ * @param int $number The number of versions to retrieve. Defaults to 10.
+ * @return array An array of file versions, where each version is an associative array containing the hash and content.
+ */
+ public function versions(int $number = 10): array {
+ $log = $this->gitService->runCommand(sprintf('log -n %d --pretty=format:%%H -- %s', $number, escapeshellarg($this->path)));
$hashes = array_filter(array_map('trim', explode("\n", $log)));
$versions = [];
foreach ($hashes as $hash) {
- $content = $this->gitService->runCommand(sprintf('show %s:%s', escapeshellarg($hash), escapeshellarg($path)));
+ $content = $this->gitService->runCommand(sprintf('show %s:%s', escapeshellarg($hash), escapeshellarg($this->path)));
$versions[] = [
'hash' => $hash,
'content' => $content,
@@ -39,4 +50,14 @@ public function versions(string $path, int $number = 10): array
return $versions;
}
+
+ /**
+ * Generates a git diff for the provided file path.
+ *
+ * @param string $path The file path for which to generate the git diff.
+ * @return string The result of the git diff command.
+ */
+ public function diff(): string {
+ return $this->gitService->runCommand(sprintf('diff HEAD -- %s', escapeshellarg($this->path)));
+ }
}
\ No newline at end of file
diff --git a/src/Utils/Git/Git.php b/src/Utils/Git/Git.php
index 8296ace5..84ccbeab 100644
--- a/src/Utils/Git/Git.php
+++ b/src/Utils/Git/Git.php
@@ -7,93 +7,71 @@ class Git
protected string $repoPath;
protected GitService $gitService;
- public function __construct(string $repoPath)
- {
+ public function __construct(string $repoPath) {
$this->repoPath = $repoPath;
$this->gitService = new GitService($repoPath);
}
- public static function dir(string $repoPath): self
- {
+ public static function dir(string $repoPath): self {
return new self($repoPath);
}
- public function add(string $path): self
- {
+ public function add(string $path): self {
$this->gitService->runCommand(sprintf('add %s', escapeshellarg($path)));
return $this;
}
- public function commit(string $message): self
- {
+ public function commit(string $message): self {
$this->gitService->runCommand(sprintf('commit -m %s', escapeshellarg($message)));
return $this;
}
- public function push(string $remote = 'origin', string $branch = 'HEAD'): self
- {
+ public function push(string $remote = 'origin', string $branch = 'HEAD'): self {
$this->gitService->runCommand(sprintf('push %s %s', escapeshellarg($remote), escapeshellarg($branch)));
return $this;
}
- public function pull(string $remote = 'origin', string $branch = 'HEAD'): self
- {
+ public function pull(string $remote = 'origin', string $branch = 'HEAD'): self {
$this->gitService->runCommand(sprintf('pull %s %s', escapeshellarg($remote), escapeshellarg($branch)));
return $this;
}
- public function branches(): Branches
- {
- return new Branches($this->gitService);
- }
-
- public function commits(): Commits
- {
- return new Commits($this->gitService);
- }
-
- public function reset(string $commit = 'HEAD', bool $hard = false): self
- {
+ public function reset(string $commit = 'HEAD', bool $hard = false): self {
$command = $hard ? 'reset --hard' : 'reset';
$this->gitService->runCommand(sprintf('%s %s', $command, escapeshellarg($commit)));
return $this;
}
- public function merge(string $branch): self
- {
- $this->gitService->runCommand(sprintf('merge %s', escapeshellarg($branch)));
- return $this;
+ public function status(): array {
+ $status = $this->gitService->runCommand('status --short');
+ return array_filter(array_map('trim', explode("\n", $status)));
}
- public function rebase(string $branch): self
- {
- $this->gitService->runCommand(sprintf('rebase %s', escapeshellarg($branch)));
- return $this;
+ public function branch(string $branchName): Branch {
+ return new Branch($branchName, $this->gitService);
}
- public function status(): array
- {
- $status = $this->gitService->runCommand('status --short');
- return array_filter(array_map('trim', explode("\n", $status)));
+ public function branches(): Branches {
+ return new Branches($this->gitService);
+ }
+
+ public function commits(): Commits {
+ return new Commits($this->gitService);
}
- public function remote(): Remote
- {
- return new Remote($this->gitService);
+ public function remote(string $remote = 'origin'): Remote {
+ return new Remote($remote, $this->gitService);
}
- public function stash(): Stash
- {
+ public function stash(): Stash {
return new Stash($this->gitService);
}
- public function diff(): Diff
- {
+ public function diff(): Diff {
return new Diff($this->gitService);
}
- public function file(): File
- {
- return new File($this->gitService);
+ public function file(string $path): File {
+ return new File($path, $this->gitService);
}
}
diff --git a/src/Utils/Git/GitService.php b/src/Utils/Git/GitService.php
index 552974ce..1ab5793d 100644
--- a/src/Utils/Git/GitService.php
+++ b/src/Utils/Git/GitService.php
@@ -8,13 +8,11 @@ class GitService
{
protected string $repoPath;
- public function __construct(string $repoPath)
- {
+ public function __construct(string $repoPath) {
$this->repoPath = $repoPath;
}
- public function runCommand(string $command): string
- {
+ public function runCommand(string $command): string {
$fullCommand = sprintf('cd %s && git %s', escapeshellarg($this->repoPath), $command);
$output = shell_exec($fullCommand);
if ($output === null) {
diff --git a/src/Utils/Git/Remote.php b/src/Utils/Git/Remote.php
index 88380340..2caef5d2 100644
--- a/src/Utils/Git/Remote.php
+++ b/src/Utils/Git/Remote.php
@@ -5,19 +5,31 @@
class Remote
{
protected GitService $gitService;
+ private string $remote;
- public function __construct(GitService $gitService)
- {
+ public function __construct(
+ string $remote,
+ GitService $gitService,
+ ) {
+ $this->remote = $remote;
$this->gitService = $gitService;
}
- public function url(): string
- {
+ public function url(): string {
return $this->gitService->runCommand('config --get remote.origin.url');
}
- public function diff(string $branch): string
- {
- return $this->gitService->runCommand(sprintf('diff origin/%s', escapeshellarg($branch)));
+ public function diff(string $branch): string {
+ return $this->gitService->runCommand(sprintf('diff %s/%s', escapeshellarg($this->remote), escapeshellarg($branch)));
+ }
+
+ public function push(string $branch = 'HEAD'): self {
+ $this->gitService->runCommand(sprintf('push %s %s', escapeshellarg($this->remote), escapeshellarg($branch)));
+ return $this;
+ }
+
+ public function pull(string $branch = 'HEAD'): self {
+ $this->gitService->runCommand(sprintf('pull %s %s', escapeshellarg($this->remote), escapeshellarg($branch)));
+ return $this;
}
}
diff --git a/src/Utils/Git/Stash.php b/src/Utils/Git/Stash.php
index da2b7323..4cd7f814 100644
--- a/src/Utils/Git/Stash.php
+++ b/src/Utils/Git/Stash.php
@@ -2,23 +2,34 @@
namespace Cognesy\Instructor\Utils\Git;
+/**
+ * Stash class provides methods to interact with Git stash functionality.
+ * It allows for saving and applying stashed changes using a GitService.
+ */
class Stash
{
protected GitService $gitService;
- public function __construct(GitService $gitService)
- {
+ public function __construct(GitService $gitService) {
$this->gitService = $gitService;
}
- public function save(): self
- {
+ /**
+ * Saves stash
+ *
+ * @return self
+ */
+ public function save(): self {
$this->gitService->runCommand('stash');
return $this;
}
- public function apply(): self
- {
+ /**
+ * Applies the latest stashed changes to the working directory.
+ *
+ * @return self Returns the current instance for method chaining.
+ */
+ public function apply(): self {
$this->gitService->runCommand('stash apply');
return $this;
}
diff --git a/src/Utils/Messages/Message.php b/src/Utils/Messages/Message.php
index f8f2cbee..1d94ecbc 100644
--- a/src/Utils/Messages/Message.php
+++ b/src/Utils/Messages/Message.php
@@ -7,6 +7,8 @@ class Message {
use Traits\Message\HandlesAccess;
use Traits\Message\HandlesTransformation;
+ public const DEFAULT_ROLE = 'user';
+
/**
* @param string $role
* @param string|array $content
diff --git a/src/Utils/Messages/Traits/Message/HandlesCreation.php b/src/Utils/Messages/Traits/Message/HandlesCreation.php
index 7f7a6e64..ad3522ed 100644
--- a/src/Utils/Messages/Traits/Message/HandlesCreation.php
+++ b/src/Utils/Messages/Traits/Message/HandlesCreation.php
@@ -10,8 +10,6 @@
trait HandlesCreation
{
- public const DEFAULT_ROLE = 'user';
-
public static function make(string $role, string|array $content) : static {
return new static($role, $content);
}
diff --git a/src/Utils/Messages/Traits/Message/HandlesTransformation.php b/src/Utils/Messages/Traits/Message/HandlesTransformation.php
index 47e9af0b..a053765e 100644
--- a/src/Utils/Messages/Traits/Message/HandlesTransformation.php
+++ b/src/Utils/Messages/Traits/Message/HandlesTransformation.php
@@ -25,23 +25,27 @@ public function toString() : string {
return $text;
}
+ public function toRoleString() : string {
+ return $this->role . ': ' . $this->toString();
+ }
+
public function toCompositeMessage() : Message {
return Message::fromArray($this->toCompositeArray());
}
public function toCompositeArray() : array {
- if ($this->isComposite()) {
- return [
+ return match($this->isComposite()) {
+ true => [
'role' => $this->role,
'content' => $this->content,
- ];
- }
- return [
- 'role' => $this->role,
- 'content' => [[
- 'type' => 'text',
- 'text' => $this->content,
- ]]
- ];
+ ],
+ default => [
+ 'role' => $this->role,
+ 'content' => [[
+ 'type' => 'text',
+ 'text' => $this->content,
+ ]]
+ ]
+ };
}
}
diff --git a/src/Utils/Messages/Traits/Messages/HandlesConversion.php b/src/Utils/Messages/Traits/Messages/HandlesConversion.php
index 6772e08b..fa64f8ad 100644
--- a/src/Utils/Messages/Traits/Messages/HandlesConversion.php
+++ b/src/Utils/Messages/Traits/Messages/HandlesConversion.php
@@ -29,7 +29,7 @@ public static function asPerRoleArray(array $messages) : array {
$role = $message['role'];
$content = [];
- if (\Cognesy\Instructor\Utils\Messages\Message::becomesComposite($message)) {
+ if (Message::becomesComposite($message)) {
$merged->appendMessage($message);
continue;
}
diff --git a/src/Utils/Messages/Traits/Messages/HandlesMutation.php b/src/Utils/Messages/Traits/Messages/HandlesMutation.php
index a1bc4dbd..80d6ce0e 100644
--- a/src/Utils/Messages/Traits/Messages/HandlesMutation.php
+++ b/src/Utils/Messages/Traits/Messages/HandlesMutation.php
@@ -8,14 +8,14 @@ trait HandlesMutation
{
public function setMessage(string|array|Message $message) : static {
$this->messages = match (true) {
- is_string($message) => [\Cognesy\Instructor\Utils\Messages\Message::fromString($message)],
- is_array($message) => [\Cognesy\Instructor\Utils\Messages\Message::fromArray($message)],
+ is_string($message) => [Message::fromString($message)],
+ is_array($message) => [Message::fromArray($message)],
default => [$message],
};
return $this;
}
- public function appendMessage(array|\Cognesy\Instructor\Utils\Messages\Message $message) : static {
+ public function appendMessage(array|Message $message) : static {
$this->messages[] = match (true) {
is_array($message) => Message::fromArray($message),
default => $message,
diff --git a/src/Utils/Messages/Traits/Messages/HandlesTransformation.php b/src/Utils/Messages/Traits/Messages/HandlesTransformation.php
index d980fbdb..ca8be9d4 100644
--- a/src/Utils/Messages/Traits/Messages/HandlesTransformation.php
+++ b/src/Utils/Messages/Traits/Messages/HandlesTransformation.php
@@ -38,6 +38,14 @@ public function exceptRoles(array $roles) : Messages {
return $messages;
}
+ public function toRoleString() : string {
+ $text = '';
+ foreach ($this->messages as $message) {
+ $text .= $message->toRoleString() . "\n";
+ }
+ return $text;
+ }
+
// INTERNAL /////////////////////////////////////////////////////////////////////////
private function mergeRolesFlat() : Messages {
diff --git a/src/Utils/Messages/Traits/RendersContent.php b/src/Utils/Messages/Traits/RendersContent.php
index 55808832..e9d94447 100644
--- a/src/Utils/Messages/Traits/RendersContent.php
+++ b/src/Utils/Messages/Traits/RendersContent.php
@@ -21,7 +21,7 @@ private function renderString(string $template, ?array $parameters) : string {
}
/**
- * @param array|\Cognesy\Instructor\Utils\Messages\Message $messages
+ * @param array|Message $messages
* @param array|null $parameters
* @return string
*/
@@ -33,7 +33,7 @@ protected function renderMessage(array|Message $message, ?array $parameters) : a
}
/**
- * @param array|\Cognesy\Instructor\Utils\Messages\Messages $messages
+ * @param array|Messages $messages
* @param array|null $parameters
* @return string
*/
diff --git a/src/Utils/Str.php b/src/Utils/Str.php
index 09f97160..6ca2315d 100644
--- a/src/Utils/Str.php
+++ b/src/Utils/Str.php
@@ -112,7 +112,13 @@ public static function when(bool $condition, string $onTrue, string $onFalse) :
};
}
- public static function limit(string $text, int $limit, string $end = '…', int $align = STR_PAD_RIGHT, bool $fit = true) : string {
+ public static function limit(
+ string $text,
+ int $limit,
+ string $cutMarker = '…',
+ int $align = STR_PAD_RIGHT,
+ bool $fit = true
+ ) : string {
$short = ($align === STR_PAD_LEFT)
? substr($text, -$limit)
: substr($text, 0, $limit);
@@ -121,14 +127,15 @@ public static function limit(string $text, int $limit, string $end = '…', int
return $text;
}
- if ($fit) {
+ $cutLength = strlen($cutMarker);
+ if ($fit && $cutLength > 0) {
return ($align === STR_PAD_LEFT)
- ? $end . substr($short,1)
- : substr($short, 0, -1) . $end;
+ ? $cutMarker . substr($short, $cutLength)
+ : substr($short, 0, -$cutLength) . $cutMarker;
}
return ($align === STR_PAD_LEFT)
- ? $end . $short
- : $short . $end;
+ ? $cutMarker . $short
+ : $short . $cutMarker;
}
}
diff --git a/src/Utils/Xml.php b/src/Utils/Xml.php
new file mode 100644
index 00000000..e121add4
--- /dev/null
+++ b/src/Utils/Xml.php
@@ -0,0 +1,150 @@
+xmlString = $xmlString;
+ }
+
+ /**
+ * Create a new instance from XML string
+ * @param string $xmlString
+ * @return self
+ */
+ public static function from(string $xmlString): self {
+ return new self($xmlString);
+ }
+
+ /**
+ * Include attributes in the resulting array
+ * @return self
+ */
+ public function withAttributes(): self {
+ $this->includeAttributes = true;
+ return $this;
+ }
+
+ /**
+ * Include root element in the resulting array
+ * @return self
+ */
+ public function withRoot(): self {
+ $this->includeRoot = true;
+ return $this;
+ }
+
+ public function wrapped(string $root = 'root'): self {
+ $this->xmlString = "<$root>{$this->xmlString}";
+ return $this;
+ }
+
+ /**
+ * Set the naming convention for the resulting array
+ * @param string $convention
+ * @return self
+ */
+ public function withNaming(string $convention): self {
+ $this->namingConvention = $convention;
+ return $this;
+ }
+
+ /**
+ * Return the array representation of the XML
+ * @return array
+ */
+ public function toArray(): array {
+ if ($this->parsedArray === null) {
+ $this->parsedArray = $this->convertXmlToArray();
+ }
+ return $this->parsedArray;
+ }
+
+ // INTERNAL ///////////////////////////////////////////////////
+
+ private function convertXmlToArray(): array {
+ if ($this->xmlString === '') {
+ return [];
+ }
+
+ $xmlElement = new SimpleXMLElement($this->xmlString, LIBXML_NOCDATA);
+ $array = $this->xmlToArray($xmlElement);
+
+ if ($this->includeRoot) {
+ return [$xmlElement->getName() => $array];
+ }
+
+ return $array;
+ }
+
+ private function xmlToArray(SimpleXMLElement $element): array|string {
+ $result = [];
+
+ // Handle attributes if required
+ if ($this->includeAttributes) {
+ foreach ($element->attributes() as $attrKey => $attrValue) {
+ $result['_attributes'][$attrKey] = (string) $attrValue;
+ }
+ }
+
+ // Handle child elements
+ foreach ($element->children() as $child) {
+ $childName = $this->sanitizeName($child->getName());
+ $childValue = $this->xmlToArray($child);
+
+ if (isset($result[$childName])) {
+ if (!is_array($result[$childName]) || !isset($result[$childName][0])) {
+ $result[$childName] = [$result[$childName]];
+ }
+ $result[$childName][] = $childValue;
+ } else {
+ $result[$childName] = $childValue;
+ }
+ }
+
+ // Handle text content or CDATA
+ $textContent = trim((string) $element);
+ if (strlen($textContent) > 0) {
+ if ($this->includeAttributes && count($result) > 0) {
+ $result['_value'] = $textContent;
+ } else {
+ return $textContent;
+ }
+ }
+
+ return $result;
+ }
+
+ /**
+ * TODO: allow conversion of names to snake_case or camelCase
+ * @param string $name
+ * @return string
+ */
+ private function sanitizeName(string $name): string {
+ return match($this->namingConvention) {
+ 'camel' => Str::camel($name),
+ 'snake' => Str::snake($name),
+ default => $name,
+ };
+ }
+}
diff --git a/tests/Feature/Extras/PromptTest.php b/tests/Feature/Extras/PromptTest.php
new file mode 100644
index 00000000..0375eed9
--- /dev/null
+++ b/tests/Feature/Extras/PromptTest.php
@@ -0,0 +1,84 @@
+toBeInstanceOf(Prompt::class);
+});
+
+it('can set a custom config', function () {
+ $config = new PromptEngineConfig();
+ $prompt = new Prompt();
+ $prompt->withConfig($config);
+ expect($prompt->config())->toBe($config);
+});
+
+it('can set template content directly', function () {
+ $content = 'template content';
+ $prompt = new Prompt();
+ $prompt->withTemplateContent($content);
+ expect($prompt->template())->toBe($content);
+});
+
+it('can set parameters for rendering', function () {
+ $params = ['key' => 'value'];
+ $prompt = new Prompt();
+ $prompt->withValues($params);
+ expect($prompt->params())->toBe($params);
+});
+
+it('can load a template by name - Blade', function () {
+ $prompt = Prompt::using('blade')->withTemplate('hello');
+ expect($prompt->template())->toContain('Hello');
+});
+
+it('can render the template - Blade', function () {
+ $prompt = Prompt::using('blade')
+ ->withTemplateContent('Hello, {{ $name }}!')
+ ->withValues(['name' => 'World']);
+ expect($prompt->toText())->toBe('Hello, World!');
+});
+
+it('can find template variables - Blade', function () {
+ $prompt = Prompt::using('blade')
+ ->withTemplateContent('Hello, {{ $name }}!')
+ ->withValues(['name' => 'World']);
+ $variables = $prompt->variables();
+ expect($variables)->toContain('name');
+});
+
+it('can convert rendered text to messages', function () {
+ $prompt = Prompt::using('blade')
+ ->withTemplateContent('Hello, {{ $name }}!')
+ ->withValues(['name' => 'World']);
+ $messages = $prompt->toMessages();
+ expect($messages)->toBeInstanceOf(Messages::class);
+});
+
+it('can load a template by name - Twig', function () {
+ $prompt = Prompt::using('twig')->withTemplate('hello');
+ expect($prompt->template())->toContain('Hello');
+});
+
+it('can render the template - Twig', function () {
+ $prompt = Prompt::using('twig')
+ ->withTemplateContent('Hello, {{ name }}!')
+ ->withValues(['name' => 'World']);
+ expect($prompt->toText())->toBe('Hello, World!');
+});
+
+it('can find template variables - Twig', function () {
+ $prompt = Prompt::using('twig')
+ ->withTemplateContent('Hello, {{ name }}!')
+ ->withValues(['name' => 'World']);
+ $variables = $prompt->variables();
+ expect($variables)->toContain('name');
+});
+
+it('can render the template using short syntax', function () {
+ $prompt = Prompt::text(name: 'hello', variables: ['name' => 'World']);
+ expect($prompt)->toBe('Hello, World!');
+});
diff --git a/tests/Feature/Utils/ConsoleTest.php b/tests/Feature/Utils/ConsoleTest.php
index c35dff7b..7c8cb5a2 100644
--- a/tests/Feature/Utils/ConsoleTest.php
+++ b/tests/Feature/Utils/ConsoleTest.php
@@ -28,7 +28,7 @@
it('truncates and appends ellipsis to long text based on maxWidth', function () {
$longText = str_repeat('A', 100);
$output = Console::columns([[-1, $longText]], 80);
- $expected = str_pad(substr($longText, 0, 79) . '…' . Color::RESET . ' ', 80);
+ $expected = str_pad(substr($longText, 0, 77) . '…' . Color::RESET . ' ', 80);
expect($output)->toBe($expected);
});
diff --git a/tests/Feature/Utils/XmlTest.php b/tests/Feature/Utils/XmlTest.php
new file mode 100644
index 00000000..413298bb
--- /dev/null
+++ b/tests/Feature/Utils/XmlTest.php
@@ -0,0 +1,119 @@
+content';
+ $xml = Xml::from($xmlString)->withAttributes();
+ $expected = [
+ '_attributes' => ['attr' => 'value'],
+ 'child' => 'content'
+ ];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('converts XML to array without attributes', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString);
+ $expected = ['child' => 'content'];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('includes root element in array', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString)->withRoot();
+ $expected = ['root' => ['child' => 'content']];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('converts names to snake_case', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString)->withNaming('snake');
+ $expected = ['child_element' => 'content'];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('converts names to camelCase', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString)->withNaming('camel');
+ $expected = ['childElement' => 'content'];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles empty XML string', function () {
+ $xmlString = '';
+ $xml = Xml::from($xmlString);
+ $expected = [];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles XML with multiple children', function () {
+ $xmlString = 'content1content2';
+ $xml = Xml::from($xmlString);
+ $expected = ['child' => ['content1', 'content2']];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles XML with multiple children with the same name', function () {
+ $xmlString = 'content1content2';
+ $xml = Xml::from($xmlString)->withNaming('snake');
+ $expected = ['child' => ['content1', 'content2']];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles CDATA in XML', function () {
+ $xmlString = 'content]]>';
+ $xml = Xml::from($xmlString)->withRoot();
+ $expected = ['root' => ['child' => 'content']];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('throws exception for invalid XML', function () {
+ $xmlString = 'content';
+ expect(fn() => Xml::from($xmlString)->toArray())->toThrow(Exception::class);
+});
+
+it('handles attributes with special characters in XML', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString)->withAttributes();
+ $expected = [
+ '_attributes' => ['attr' => 'value & more'],
+ 'child' => 'content'
+ ];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('wraps XML string with specified root element', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString)->withRoot()->wrapped('wrapper');
+ $expected = ['wrapper' => ['child' => 'content']];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles special characters in XML', function () {
+ $xmlString = 'content & more content';
+ $xml = Xml::from($xmlString);
+ $expected = ['child' => 'content & more content'];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles empty elements in XML', function () {
+ $xmlString = '';
+ $xml = Xml::from($xmlString);
+ $expected = ['child' => []];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles nested elements in XML', function () {
+ $xmlString = 'content';
+ $xml = Xml::from($xmlString);
+ $expected = ['parent' => ['child' => 'content']];
+ expect($xml->toArray())->toEqual($expected);
+});
+
+it('handles mixed content in XML', function () {
+ $xmlString = 'textcontentmore text';
+ $xml = Xml::from($xmlString)->withRoot()->toArray();
+ $expected = ['root' => ['textmore text', ['child' => 'content']]];
+ expect($xml)->toEqual($expected);
+})->skip();
\ No newline at end of file