Skip to content

Commit 126df5b

Browse files
AnkushMalakerAnkushMalaker
committed
Add mock providers and update testing workflows for API-independent execution
- Introduced `MockLLMProvider` and `MockTranscriptionProvider` to facilitate testing without external API dependencies, allowing for consistent and controlled test environments. - Created `run-no-api-tests.sh` script to execute tests that do not require API keys, ensuring separation of API-dependent and independent tests. - Updated Robot Framework test configurations to utilize mock services, enhancing test reliability and reducing external dependencies. - Modified existing test workflows to include new configurations and ensure proper handling of results for tests excluding API keys. - Added `mock-services.yml` configuration to disable external API services while maintaining core functionality for testing purposes. - Enhanced documentation to reflect the new tagging system for tests requiring API keys, improving clarity on test execution requirements.
1 parent d143fe7 commit 126df5b

File tree

10 files changed

+1063
-57
lines changed

10 files changed

+1063
-57
lines changed

‎.github/workflows/full-tests-with-api.yml‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
name: Robot Framework Tests (Full - With API Keys)
2+
3+
on:
4+
push:
5+
branches:
6+
- dev
7+
- main
8+
paths:
9+
- 'tests/**/*.robot'
10+
- 'tests/**/*.py'
11+
- 'backends/advanced/src/**'
12+
- '.github/workflows/full-tests-with-api.yml'
13+
workflow_dispatch: # Allow manual triggering
14+
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
issues: write
19+
pages: write
20+
id-token: write
21+
22+
jobs:
23+
full-robot-tests:
24+
runs-on: ubuntu-latest
25+
timeout-minutes: 30
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
31+
- name: Verify required secrets
32+
env:
33+
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
34+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
35+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
36+
run: |
37+
echo "Verifying required secrets..."
38+
if [ -z "$DEEPGRAM_API_KEY" ]; then
39+
echo "❌ ERROR: DEEPGRAM_API_KEY secret is not set"
40+
exit 1
41+
fi
42+
if [ -z "$OPENAI_API_KEY" ]; then
43+
echo "❌ ERROR: OPENAI_API_KEY secret is not set"
44+
exit 1
45+
fi
46+
if [ -z "$HF_TOKEN" ]; then
47+
echo "⚠️ WARNING: HF_TOKEN secret is not set (speaker recognition will be disabled)"
48+
else
49+
echo "✓ HF_TOKEN is set (length: ${#HF_TOKEN})"
50+
fi
51+
echo "✓ DEEPGRAM_API_KEY is set (length: ${#DEEPGRAM_API_KEY})"
52+
echo "✓ OPENAI_API_KEY is set (length: ${#OPENAI_API_KEY})"
53+
echo "✓ Required secrets verified"
54+
55+
- name: Set up Docker Buildx
56+
uses: docker/setup-buildx-action@v3
57+
with:
58+
driver-opts: |
59+
image=moby/buildkit:latest
60+
network=host
61+
62+
- name: Cache Docker layers
63+
uses: actions/cache@v4
64+
with:
65+
path: /tmp/.buildx-cache
66+
key: ${{ runner.os }}-buildx-${{ hashFiles('backends/advanced/Dockerfile', 'backends/advanced/pyproject.toml') }}
67+
restore-keys: |
68+
${{ runner.os }}-buildx-
69+
70+
- name: Set up Python
71+
uses: actions/setup-python@v5
72+
with:
73+
python-version: "3.12"
74+
75+
- name: Install uv
76+
uses: astral-sh/setup-uv@v4
77+
with:
78+
version: "latest"
79+
80+
- name: Install Robot Framework and dependencies
81+
run: |
82+
uv pip install --system robotframework robotframework-requests python-dotenv websockets
83+
84+
- name: Create test config.yml
85+
run: |
86+
echo "Copying test configuration file..."
87+
mkdir -p config
88+
cp tests/configs/deepgram-openai.yml config/config.yml
89+
echo "✓ Test config.yml created from tests/configs/deepgram-openai.yml"
90+
ls -lh config/config.yml
91+
92+
- name: Create plugins.yml from template
93+
run: |
94+
echo "Creating plugins.yml from template..."
95+
if [ -f "config/plugins.yml.template" ]; then
96+
cp config/plugins.yml.template config/plugins.yml
97+
echo "✓ plugins.yml created from template"
98+
ls -lh config/plugins.yml
99+
else
100+
echo "❌ ERROR: config/plugins.yml.template not found"
101+
exit 1
102+
fi
103+
104+
- name: Run Full Robot Framework tests
105+
working-directory: tests
106+
env:
107+
# Required for test runner script
108+
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
109+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
110+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
111+
CLEANUP_CONTAINERS: "false" # Don't cleanup in CI - handled by workflow
112+
run: |
113+
# Use the full test script (includes all tests with API keys)
114+
./run-robot-tests.sh
115+
TEST_EXIT_CODE=$?
116+
echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV
117+
exit 0 # Don't fail here, we'll fail at the end after uploading artifacts
118+
119+
- name: Show service logs
120+
if: always()
121+
working-directory: backends/advanced
122+
run: |
123+
echo "=== Backend Logs (last 50 lines) ==="
124+
docker compose -f docker-compose-test.yml logs --tail=50 chronicle-backend-test
125+
echo ""
126+
echo "=== Worker Logs (last 50 lines) ==="
127+
docker compose -f docker-compose-test.yml logs --tail=50 workers-test
128+
129+
- name: Check if test results exist
130+
if: always()
131+
id: check_results
132+
run: |
133+
if [ -f tests/results/output.xml ]; then
134+
echo "results_exist=true" >> $GITHUB_OUTPUT
135+
else
136+
echo "results_exist=false" >> $GITHUB_OUTPUT
137+
echo "⚠️ No test results found in tests/results/"
138+
ls -la tests/results/ || echo "Results directory doesn't exist"
139+
fi
140+
141+
- name: Upload Robot Framework HTML reports
142+
if: always() && steps.check_results.outputs.results_exist == 'true'
143+
uses: actions/upload-artifact@v4
144+
with:
145+
name: robot-test-reports-html-full
146+
path: |
147+
tests/results/report.html
148+
tests/results/log.html
149+
retention-days: 30
150+
151+
- name: Publish HTML Report as GitHub Pages artifact
152+
if: always() && steps.check_results.outputs.results_exist == 'true'
153+
uses: actions/upload-pages-artifact@v3
154+
with:
155+
path: tests/results
156+
157+
- name: Deploy to GitHub Pages
158+
if: always() && steps.check_results.outputs.results_exist == 'true'
159+
uses: actions/deploy-pages@v4
160+
id: deployment
161+
162+
- name: Generate test summary
163+
if: always() && steps.check_results.outputs.results_exist == 'true'
164+
id: test_summary
165+
run: |
166+
# Parse test results
167+
python3 << 'PYTHON_SCRIPT' > test_summary.txt
168+
import xml.etree.ElementTree as ET
169+
tree = ET.parse('tests/results/output.xml')
170+
root = tree.getroot()
171+
stats = root.find('.//total/stat')
172+
if stats is not None:
173+
passed = stats.get("pass", "0")
174+
failed = stats.get("fail", "0")
175+
total = int(passed) + int(failed)
176+
print(f"PASSED={passed}")
177+
print(f"FAILED={failed}")
178+
print(f"TOTAL={total}")
179+
PYTHON_SCRIPT
180+
181+
# Source the variables
182+
source test_summary.txt
183+
184+
# Set outputs
185+
echo "passed=$PASSED" >> $GITHUB_OUTPUT
186+
echo "failed=$FAILED" >> $GITHUB_OUTPUT
187+
echo "total=$TOTAL" >> $GITHUB_OUTPUT
188+
189+
- name: Upload Robot Framework XML output
190+
if: always() && steps.check_results.outputs.results_exist == 'true'
191+
uses: actions/upload-artifact@v4
192+
with:
193+
name: robot-test-results-xml-full
194+
path: tests/results/output.xml
195+
retention-days: 30
196+
197+
- name: Upload logs on failure
198+
if: failure()
199+
uses: actions/upload-artifact@v4
200+
with:
201+
name: robot-test-logs-full
202+
path: |
203+
backends/advanced/.env
204+
tests/setup/.env.test
205+
retention-days: 7
206+
207+
- name: Display test results summary
208+
if: always()
209+
run: |
210+
if [ -f tests/results/output.xml ]; then
211+
echo "Full test results generated successfully (With API Keys)"
212+
echo "========================================"
213+
python3 << 'PYTHON_SCRIPT'
214+
import xml.etree.ElementTree as ET
215+
tree = ET.parse('tests/results/output.xml')
216+
root = tree.getroot()
217+
stats = root.find('.//total/stat')
218+
if stats is not None:
219+
passed = stats.get("pass", "0")
220+
failed = stats.get("fail", "0")
221+
print(f'✅ Passed: {passed}')
222+
print(f'❌ Failed: {failed}')
223+
print(f'📊 Total: {int(passed) + int(failed)}')
224+
PYTHON_SCRIPT
225+
echo "========================================"
226+
echo ""
227+
echo "ℹ️ Full test suite including API-dependent tests"
228+
echo ""
229+
echo "📊 FULL TEST REPORTS AVAILABLE:"
230+
echo " 1. Go to the 'Summary' tab at the top of this page"
231+
echo " 2. Scroll down to 'Artifacts' section"
232+
echo " 3. Download 'robot-test-reports-html-full'"
233+
echo " 4. Extract and open report.html or log.html in your browser"
234+
echo ""
235+
echo "The HTML reports provide:"
236+
echo " - report.html: Executive summary with statistics"
237+
echo " - log.html: Detailed step-by-step execution log"
238+
echo ""
239+
fi
240+
241+
- name: Cleanup
242+
if: always()
243+
working-directory: backends/advanced
244+
run: |
245+
docker compose -f docker-compose-test.yml down -v
246+
247+
- name: Fail workflow if tests failed
248+
if: always()
249+
run: |
250+
if [ "${{ env.test_exit_code }}" != "0" ]; then
251+
echo "❌ Tests failed with exit code ${{ env.test_exit_code }}"
252+
exit 1
253+
else
254+
echo "✅ All tests passed"
255+
fi

0 commit comments

Comments
 (0)