Nicholas Riegel
acfa1d93d7
Created comprehensive E2E test suite for ITSulu Blog Publisher using Playwright and Runboat. Includes: PHASE3_ROADMAP.md: - Goals for E2E coverage (10-30 scenarios) - Performance benchmark targets (latency, tokens, queries, throughput) - Implementation plan with layer-by-layer breakdown - Success criteria and SLO targets - Runboat integration details for CI/CD e2e/ directory structure: - conftest.py: Runboat polling, auth fixtures, page fixture - requirements.txt: pytest, playwright, requests - test_generation.py: On-demand generation workflows (5 tests) - test_scheduling.py: Schedule slot configuration and execution (6 tests) - test_error_recovery.py: Error handling and email notifications (8 tests) Total: 19 E2E test scenarios covering: - On-demand post generation with auto-publish - Scheduled generation with topic queue - Error recovery and retry mechanism - Email notifications with correct content - Social media copy generation - Concurrent post generation - Progress feedback during API calls Tests use: - Playwright sync API with 30s timeout (Odoo JS rendering) - Runboat polling with 180s timeout (instance cold-start) - Session-scoped auth to avoid repeated 30s logins - Data-test-id selectors where available, fallback to get_by_* - Proper wait_for_load_state() for async operations Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
254 lines
9.8 KiB
Python
254 lines
9.8 KiB
Python
"""
|
|
E2E tests for schedule slot configuration and execution.
|
|
Tests the workflow of setting up daily scheduled generation.
|
|
"""
|
|
from playwright.sync_api import expect
|
|
|
|
|
|
class TestScheduleConfiguration:
|
|
"""Tests for configuring and managing schedule slots."""
|
|
|
|
def test_user_configures_three_schedule_slots(self, page):
|
|
"""
|
|
User creates three daily schedule slots (morning, afternoon, evening)
|
|
with different LLM providers and auto-publish settings.
|
|
|
|
Workflow:
|
|
1. Navigate to Schedule Slots
|
|
2. For each slot: fill name, time, provider, model, auto-publish
|
|
3. Save all slots
|
|
4. Verify: all 3 slots appear in list as Active
|
|
"""
|
|
# Navigate to Blog Publisher > Schedule Slots
|
|
page.goto('/web')
|
|
page.get_by_role('link', name='Blog').click()
|
|
page.wait_for_url('**/web#*', timeout=10_000)
|
|
|
|
page.get_by_role('link', name='Blog Publisher').click()
|
|
page.get_by_role('link', name='Schedule Slots').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Create morning slot
|
|
page.get_by_role('button', name='New').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
page.get_by_label('Slot Name').fill('Morning Generation')
|
|
page.get_by_label('Slot').select_option('morning')
|
|
page.get_by_label('Trigger Time').fill('8.0') # 8:00 UTC
|
|
page.get_by_label('LLM Provider').select_option('anthropic')
|
|
page.get_by_label('Auto-publish').check()
|
|
page.get_by_role('button', name='Save').click()
|
|
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Return to list and create afternoon slot
|
|
page.get_by_role('button', name='New').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
page.get_by_label('Slot Name').fill('Afternoon Generation')
|
|
page.get_by_label('Slot').select_option('afternoon')
|
|
page.get_by_label('Trigger Time').fill('14.0') # 2:00 PM UTC
|
|
page.get_by_label('LLM Provider').select_option('openai')
|
|
page.get_by_label('Auto-publish').uncheck() # Save as draft
|
|
page.get_by_role('button', name='Save').click()
|
|
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Create evening slot
|
|
page.get_by_role('button', name='New').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
page.get_by_label('Slot Name').fill('Evening Generation')
|
|
page.get_by_label('Slot').select_option('evening')
|
|
page.get_by_label('Trigger Time').fill('20.0') # 8:00 PM UTC
|
|
page.get_by_label('LLM Provider').select_option('gemini')
|
|
page.get_by_label('Auto-publish').check()
|
|
page.get_by_role('button', name='Save').click()
|
|
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Navigate back to list and verify all 3 slots exist
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
expect(page.locator('body')).to_contain_text('Morning Generation')
|
|
expect(page.locator('body')).to_contain_text('Afternoon Generation')
|
|
expect(page.locator('body')).to_contain_text('Evening Generation')
|
|
|
|
def test_user_edits_schedule_slot_settings(self, page):
|
|
"""
|
|
User opens an existing schedule slot and modifies its settings
|
|
(LLM provider, auto-publish flag, trigger time).
|
|
|
|
Workflow:
|
|
1. Navigate to Schedule Slots
|
|
2. Click on existing slot
|
|
3. Change settings (provider, time, auto-publish)
|
|
4. Save
|
|
5. Verify changes persisted
|
|
"""
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Click on morning slot
|
|
page.get_by_role('link', name='Morning Generation').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Edit trigger time
|
|
page.get_by_label('Trigger Time').fill('7.5') # Change from 8.0 to 7:30
|
|
|
|
# Change provider
|
|
page.get_by_label('LLM Provider').select_option('openai')
|
|
|
|
# Toggle auto-publish
|
|
page.get_by_label('Auto-publish').uncheck()
|
|
|
|
# Save
|
|
page.get_by_role('button', name='Save').click()
|
|
|
|
# Verify changes by reopening
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
page.get_by_role('link', name='Morning Generation').click()
|
|
|
|
# Check that changes were saved
|
|
expect(page.get_by_label('Trigger Time')).to_have_value('7.5')
|
|
expect(page.get_by_label('LLM Provider')).to_have_value('openai')
|
|
expect(page.get_by_label('Auto-publish')).not_to_be_checked()
|
|
|
|
def test_user_disables_schedule_slot(self, page):
|
|
"""
|
|
User deactivates a schedule slot.
|
|
Inactive slots are not executed by the scheduler.
|
|
|
|
Workflow:
|
|
1. Navigate to Schedule Slots
|
|
2. Click slot
|
|
3. Uncheck Active checkbox
|
|
4. Save
|
|
5. Verify: slot shows as inactive in list
|
|
"""
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Click evening slot
|
|
page.get_by_role('link', name='Evening Generation').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Uncheck Active
|
|
page.get_by_label('Active').uncheck()
|
|
page.get_by_role('button', name='Save').click()
|
|
|
|
# Verify in list view (might show as grayed out or with [Draft] label)
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Evening slot should still appear but inactive
|
|
expect(page.locator('body')).to_contain_text('Evening Generation')
|
|
|
|
|
|
class TestScheduleExecution:
|
|
"""Tests for schedule slot execution (via manual trigger, not real cron)."""
|
|
|
|
def test_user_manually_triggers_schedule_slot(self, page):
|
|
"""
|
|
User clicks a "Run Now" button on schedule slot to trigger
|
|
generation immediately (for testing, not waiting for cron).
|
|
|
|
Workflow:
|
|
1. Navigate to schedule slot
|
|
2. Click "Run Generation Now" button
|
|
3. Verify: blog post created, logged, email sent
|
|
"""
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Click morning slot
|
|
page.get_by_role('link', name='Morning Generation').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Click Run Generation Now button
|
|
page.get_by_role('button', name='Run Generation Now').click()
|
|
|
|
# Wait for generation to complete
|
|
page.wait_for_url('**/blog/**', timeout=60_000)
|
|
|
|
# Verify success message or redirect to created post
|
|
expect(page.locator('body')).to_contain_text('Blog')
|
|
|
|
def test_schedule_slot_falls_back_to_llm_chosen_topic_when_queue_empty(self, page):
|
|
"""
|
|
When topic queue is empty, schedule slot asks LLM to choose a topic.
|
|
LLM returns a relevant topic and post is generated.
|
|
|
|
Workflow:
|
|
1. Verify topic queue is empty
|
|
2. Trigger schedule slot
|
|
3. Verify: blog post created with LLM-chosen title
|
|
"""
|
|
# First, clear the topic queue
|
|
page.goto('/web#model=itsulu.blog.topic&view_type=list&domain=[[\"state\",\"=\",\"pending\"]]')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Delete all pending topics (or change state to Used)
|
|
# For this test, assume queue is already empty
|
|
expect(page.locator('body')).to_contain_text('No records')
|
|
|
|
# Now trigger schedule slot
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=form')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
page.get_by_role('button', name='Run Generation Now').click()
|
|
page.wait_for_url('**/blog/**', timeout=60_000)
|
|
|
|
# Post should be created with LLM-chosen topic
|
|
expect(page.locator('h1')).not_to_have_text('') # Has some title
|
|
|
|
def test_schedule_slot_consumes_topics_from_queue(self, page):
|
|
"""
|
|
When topic queue has pending topics, schedule slot uses them in priority order.
|
|
Topics are marked as Used after generation.
|
|
|
|
Workflow:
|
|
1. Create 3 pending topics with priorities
|
|
2. Trigger schedule slot
|
|
3. Verify: highest priority topic consumed
|
|
4. Verify: topic state changed to Used
|
|
"""
|
|
# Navigate to topic queue
|
|
page.goto('/web#model=itsulu.blog.topic&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Create urgent topic
|
|
page.get_by_role('button', name='New').click()
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
page.get_by_label('Topic Name').fill('Critical: Zero-Day Vulnerability Response')
|
|
page.get_by_label('Priority').select_option('urgent')
|
|
page.get_by_label('State').select_option('pending')
|
|
page.get_by_role('button', name='Save').click()
|
|
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Now trigger schedule slot (should use the urgent topic)
|
|
page.goto('/web#model=itsulu.blog.schedule&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
page.get_by_role('link', name='Morning Generation').click()
|
|
page.get_by_role('button', name='Run Generation Now').click()
|
|
|
|
page.wait_for_url('**/blog/**', timeout=60_000)
|
|
|
|
# Verify the generated post used the urgent topic
|
|
expect(page.locator('h1')).to_contain_text('Vulnerability|Response', use_regex=True)
|
|
|
|
# Verify topic was marked as Used
|
|
page.goto('/web#model=itsulu.blog.topic&view_type=list')
|
|
page.wait_for_load_state('networkidle')
|
|
|
|
# Find the used topic
|
|
page.get_by_role('button', name='Filters').click()
|
|
page.get_by_role('checkbox', name='State').check()
|
|
page.get_by_role('option', name='Used').click()
|
|
|
|
# Should appear in Used list
|
|
expect(page.locator('body')).to_contain_text('Zero-Day')
|