4479c9ef91
Phase 3 runtime testing has been completed with critical findings: - Microsoft MCP SDK is registered but NOT actually used at runtime - Application uses custom HTTP-based MCP implementation instead of SDK's stdio - SDK tools (Ping, GetServerTime, GetProjectInfo) discovered but not exposed - Requires architecture decision: Remove SDK, Use SDK properly, or Hybrid approach Test artifacts: - Complete test report with detailed analysis - Summary document for quick reference - Runtime test scripts (PowerShell) - API key creation utilities (SQL + PowerShell) Key findings: - Transport mismatch: SDK expects stdio, app uses HTTP - Tool discovery works but not integrated with custom handler - Cannot verify DI in SDK tools (tools never called) - Claude Desktop integration blocked (requires stdio) Next steps: 1. Make architecture decision (Remove/Use/Hybrid) 2. Either remove SDK or implement stdio transport 3. Bridge SDK tools to custom handler if keeping SDK Test Status: Phase 3 Complete (Blocked on architecture decision) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
394 lines
13 KiB
PowerShell
394 lines
13 KiB
PowerShell
# MCP Runtime Validation Test Script
|
|
# Phase 3: Runtime Testing and Verification
|
|
|
|
param(
|
|
[string]$BaseUrl = "http://localhost:5000",
|
|
[string]$McpEndpoint = "/mcp",
|
|
[int]$StartupWaitSeconds = 10
|
|
)
|
|
|
|
$ErrorActionPreference = "Continue"
|
|
$ProgressPreference = "SilentlyContinue"
|
|
|
|
Write-Host "=====================================" -ForegroundColor Cyan
|
|
Write-Host "MCP Runtime Validation Test" -ForegroundColor Cyan
|
|
Write-Host "=====================================" -ForegroundColor Cyan
|
|
Write-Host ""
|
|
|
|
# Test Results
|
|
$TestResults = @{
|
|
Timestamp = Get-Date
|
|
BaseUrl = $BaseUrl
|
|
Tests = @()
|
|
}
|
|
|
|
function Add-TestResult {
|
|
param(
|
|
[string]$TestName,
|
|
[string]$Status,
|
|
[object]$Details
|
|
)
|
|
|
|
$result = @{
|
|
TestName = $TestName
|
|
Status = $Status
|
|
Details = $Details
|
|
Timestamp = Get-Date
|
|
}
|
|
|
|
$TestResults.Tests += $result
|
|
|
|
$color = switch ($Status) {
|
|
"PASS" { "Green" }
|
|
"FAIL" { "Red" }
|
|
"WARN" { "Yellow" }
|
|
"INFO" { "Cyan" }
|
|
default { "White" }
|
|
}
|
|
|
|
Write-Host "[$Status] $TestName" -ForegroundColor $color
|
|
if ($Details) {
|
|
Write-Host " Details: $Details" -ForegroundColor Gray
|
|
}
|
|
}
|
|
|
|
function Test-McpEndpoint {
|
|
param(
|
|
[string]$Url,
|
|
[string]$Method,
|
|
[object]$Body
|
|
)
|
|
|
|
try {
|
|
$bodyJson = if ($Body) { $Body | ConvertTo-Json -Depth 10 } else { $null }
|
|
|
|
$params = @{
|
|
Uri = $Url
|
|
Method = "POST"
|
|
ContentType = "application/json"
|
|
Body = $bodyJson
|
|
TimeoutSec = 10
|
|
}
|
|
|
|
$response = Invoke-WebRequest @params -ErrorAction Stop
|
|
|
|
return @{
|
|
Success = $true
|
|
StatusCode = $response.StatusCode
|
|
Content = $response.Content | ConvertFrom-Json
|
|
RawContent = $response.Content
|
|
}
|
|
}
|
|
catch {
|
|
return @{
|
|
Success = $false
|
|
Error = $_.Exception.Message
|
|
StatusCode = $_.Exception.Response.StatusCode.value__
|
|
}
|
|
}
|
|
}
|
|
|
|
# Test 1: Check if application is running
|
|
Write-Host "`n1. Checking if application is running..." -ForegroundColor Yellow
|
|
|
|
try {
|
|
$healthCheck = Invoke-WebRequest -Uri "$BaseUrl/health" -Method GET -TimeoutSec 5 -ErrorAction Stop
|
|
Add-TestResult -TestName "Application Health Check" -Status "PASS" -Details "Application is running on $BaseUrl"
|
|
}
|
|
catch {
|
|
Add-TestResult -TestName "Application Health Check" -Status "FAIL" -Details "Application not responding: $($_.Exception.Message)"
|
|
Write-Host "`nPlease start the application first:" -ForegroundColor Red
|
|
Write-Host " cd colaflow-api/src/ColaFlow.API" -ForegroundColor Yellow
|
|
Write-Host " dotnet run" -ForegroundColor Yellow
|
|
exit 1
|
|
}
|
|
|
|
# Test 2: Discover MCP endpoints
|
|
Write-Host "`n2. Discovering MCP endpoints..." -ForegroundColor Yellow
|
|
|
|
$possibleEndpoints = @(
|
|
"/mcp",
|
|
"/api/mcp",
|
|
"/.well-known/mcp",
|
|
"/mcp/sse",
|
|
"/mcp/ws"
|
|
)
|
|
|
|
$discoveredEndpoint = $null
|
|
foreach ($endpoint in $possibleEndpoints) {
|
|
$url = "$BaseUrl$endpoint"
|
|
try {
|
|
$response = Invoke-WebRequest -Uri $url -Method POST -ContentType "application/json" -Body '{"jsonrpc":"2.0","id":0,"method":"ping"}' -TimeoutSec 5 -ErrorAction Stop
|
|
$discoveredEndpoint = $endpoint
|
|
Add-TestResult -TestName "Endpoint Discovery" -Status "PASS" -Details "Found MCP endpoint at $endpoint"
|
|
break
|
|
}
|
|
catch {
|
|
# Endpoint not found, try next
|
|
}
|
|
}
|
|
|
|
if (-not $discoveredEndpoint) {
|
|
Add-TestResult -TestName "Endpoint Discovery" -Status "WARN" -Details "No HTTP endpoint found. SDK might be using stdio transport."
|
|
Write-Host "`nNote: Microsoft MCP SDK might be configured for stdio transport (Claude Desktop)." -ForegroundColor Yellow
|
|
}
|
|
|
|
$McpUrl = "$BaseUrl$discoveredEndpoint"
|
|
|
|
# Test 3: MCP Initialize
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n3. Testing MCP Initialize..." -ForegroundColor Yellow
|
|
|
|
$initRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 1
|
|
method = "initialize"
|
|
params = @{
|
|
protocolVersion = "2024-11-05"
|
|
capabilities = @{}
|
|
clientInfo = @{
|
|
name = "test-client"
|
|
version = "1.0.0"
|
|
}
|
|
}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "initialize" -Body $initRequest
|
|
|
|
if ($result.Success) {
|
|
Add-TestResult -TestName "MCP Initialize" -Status "PASS" -Details "Protocol version: $($result.Content.result.protocolVersion)"
|
|
Write-Host " Server Info: $($result.Content.result.serverInfo.name) v$($result.Content.result.serverInfo.version)" -ForegroundColor Gray
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "MCP Initialize" -Status "FAIL" -Details $result.Error
|
|
}
|
|
}
|
|
|
|
# Test 4: List Tools
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n4. Testing tools/list..." -ForegroundColor Yellow
|
|
|
|
$listToolsRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 2
|
|
method = "tools/list"
|
|
params = @{}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "tools/list" -Body $listToolsRequest
|
|
|
|
if ($result.Success) {
|
|
$tools = $result.Content.result.tools
|
|
Add-TestResult -TestName "List Tools" -Status "PASS" -Details "Found $($tools.Count) tools"
|
|
|
|
Write-Host " Available Tools:" -ForegroundColor Gray
|
|
foreach ($tool in $tools) {
|
|
Write-Host " - $($tool.name): $($tool.description)" -ForegroundColor Gray
|
|
}
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "List Tools" -Status "FAIL" -Details $result.Error
|
|
}
|
|
}
|
|
|
|
# Test 5: Call Ping Tool
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n5. Testing tools/call (Ping)..." -ForegroundColor Yellow
|
|
|
|
$callPingRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 3
|
|
method = "tools/call"
|
|
params = @{
|
|
name = "Ping"
|
|
arguments = @{}
|
|
}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "tools/call" -Body $callPingRequest
|
|
|
|
if ($result.Success) {
|
|
Add-TestResult -TestName "Call Ping Tool" -Status "PASS" -Details "Response: $($result.Content.result.content[0].text)"
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "Call Ping Tool" -Status "FAIL" -Details $result.Error
|
|
}
|
|
}
|
|
|
|
# Test 6: Call GetServerTime Tool (with DI Logger)
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n6. Testing tools/call (GetServerTime) - Verifying DI..." -ForegroundColor Yellow
|
|
|
|
$callGetServerTimeRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 4
|
|
method = "tools/call"
|
|
params = @{
|
|
name = "GetServerTime"
|
|
arguments = @{}
|
|
}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "tools/call" -Body $callGetServerTimeRequest
|
|
|
|
if ($result.Success) {
|
|
Add-TestResult -TestName "Call GetServerTime Tool (DI Test)" -Status "PASS" -Details "Response: $($result.Content.result.content[0].text)"
|
|
Write-Host " Note: Check server logs to verify ILogger was injected successfully" -ForegroundColor Gray
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "Call GetServerTime Tool (DI Test)" -Status "FAIL" -Details $result.Error
|
|
}
|
|
}
|
|
|
|
# Test 7: Call GetProjectInfo Tool
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n7. Testing tools/call (GetProjectInfo)..." -ForegroundColor Yellow
|
|
|
|
$callGetProjectInfoRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 5
|
|
method = "tools/call"
|
|
params = @{
|
|
name = "GetProjectInfo"
|
|
arguments = @{
|
|
projectId = "test-project-123"
|
|
}
|
|
}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "tools/call" -Body $callGetProjectInfoRequest
|
|
|
|
if ($result.Success) {
|
|
Add-TestResult -TestName "Call GetProjectInfo Tool" -Status "PASS" -Details "Response: $($result.Content.result.content[0].text)"
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "Call GetProjectInfo Tool" -Status "FAIL" -Details $result.Error
|
|
}
|
|
}
|
|
|
|
# Test 8: List Resources
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n8. Testing resources/list..." -ForegroundColor Yellow
|
|
|
|
$listResourcesRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 6
|
|
method = "resources/list"
|
|
params = @{}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "resources/list" -Body $listResourcesRequest
|
|
|
|
if ($result.Success) {
|
|
$resources = $result.Content.result.resources
|
|
Add-TestResult -TestName "List Resources" -Status "PASS" -Details "Found $($resources.Count) resources"
|
|
|
|
if ($resources.Count -gt 0) {
|
|
Write-Host " Available Resources:" -ForegroundColor Gray
|
|
foreach ($resource in $resources) {
|
|
Write-Host " - $($resource.uri): $($resource.name)" -ForegroundColor Gray
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "List Resources" -Status "FAIL" -Details $result.Error
|
|
}
|
|
}
|
|
|
|
# Test 9: Error Handling
|
|
if ($discoveredEndpoint) {
|
|
Write-Host "`n9. Testing error handling..." -ForegroundColor Yellow
|
|
|
|
$invalidRequest = @{
|
|
jsonrpc = "2.0"
|
|
id = 7
|
|
method = "tools/call"
|
|
params = @{
|
|
name = "NonExistentTool"
|
|
arguments = @{}
|
|
}
|
|
}
|
|
|
|
$result = Test-McpEndpoint -Url $McpUrl -Method "tools/call" -Body $invalidRequest
|
|
|
|
if ($result.Success -eq $false) {
|
|
Add-TestResult -TestName "Error Handling (Invalid Tool)" -Status "PASS" -Details "Correctly returned error"
|
|
}
|
|
else {
|
|
Add-TestResult -TestName "Error Handling (Invalid Tool)" -Status "WARN" -Details "Should have returned error for non-existent tool"
|
|
}
|
|
}
|
|
|
|
# Test Summary
|
|
Write-Host "`n=====================================" -ForegroundColor Cyan
|
|
Write-Host "Test Summary" -ForegroundColor Cyan
|
|
Write-Host "=====================================" -ForegroundColor Cyan
|
|
|
|
$passCount = ($TestResults.Tests | Where-Object { $_.Status -eq "PASS" }).Count
|
|
$failCount = ($TestResults.Tests | Where-Object { $_.Status -eq "FAIL" }).Count
|
|
$warnCount = ($TestResults.Tests | Where-Object { $_.Status -eq "WARN" }).Count
|
|
|
|
Write-Host "Total Tests: $($TestResults.Tests.Count)" -ForegroundColor White
|
|
Write-Host "Passed: $passCount" -ForegroundColor Green
|
|
Write-Host "Failed: $failCount" -ForegroundColor Red
|
|
Write-Host "Warnings: $warnCount" -ForegroundColor Yellow
|
|
|
|
# Save results to JSON
|
|
$reportPath = "c:\Users\yaoji\git\ColaCoder\product-master\scripts\mcp-runtime-test-results.json"
|
|
$TestResults | ConvertTo-Json -Depth 10 | Out-File -FilePath $reportPath -Encoding UTF8
|
|
Write-Host "`nDetailed results saved to: $reportPath" -ForegroundColor Gray
|
|
|
|
# Key Questions
|
|
Write-Host "`n=====================================" -ForegroundColor Cyan
|
|
Write-Host "Key Questions to Answer" -ForegroundColor Cyan
|
|
Write-Host "=====================================" -ForegroundColor Cyan
|
|
|
|
Write-Host "1. SDK Transport Layer:" -ForegroundColor Yellow
|
|
if ($discoveredEndpoint) {
|
|
Write-Host " HTTP endpoint found at: $discoveredEndpoint" -ForegroundColor Green
|
|
}
|
|
else {
|
|
Write-Host " No HTTP endpoint found - likely using stdio" -ForegroundColor Yellow
|
|
}
|
|
|
|
Write-Host "`n2. Endpoint URL:" -ForegroundColor Yellow
|
|
Write-Host " $McpUrl" -ForegroundColor White
|
|
|
|
Write-Host "`n3. Tools Discovery:" -ForegroundColor Yellow
|
|
$toolTest = $TestResults.Tests | Where-Object { $_.TestName -eq "List Tools" }
|
|
if ($toolTest -and $toolTest.Status -eq "PASS") {
|
|
Write-Host " Tools correctly discovered: $($toolTest.Details)" -ForegroundColor Green
|
|
}
|
|
else {
|
|
Write-Host " Tools discovery failed" -ForegroundColor Red
|
|
}
|
|
|
|
Write-Host "`n4. Dependency Injection:" -ForegroundColor Yellow
|
|
$diTest = $TestResults.Tests | Where-Object { $_.TestName -eq "Call GetServerTime Tool (DI Test)" }
|
|
if ($diTest -and $diTest.Status -eq "PASS") {
|
|
Write-Host " DI appears to be working (check server logs for ILogger)" -ForegroundColor Green
|
|
}
|
|
else {
|
|
Write-Host " DI test failed or not executed" -ForegroundColor Red
|
|
}
|
|
|
|
Write-Host "`n5. Performance:" -ForegroundColor Yellow
|
|
Write-Host " Check response times in detailed results" -ForegroundColor Gray
|
|
|
|
Write-Host "`n=====================================" -ForegroundColor Cyan
|
|
Write-Host "Next Steps" -ForegroundColor Cyan
|
|
Write-Host "=====================================" -ForegroundColor Cyan
|
|
|
|
if (-not $discoveredEndpoint) {
|
|
Write-Host "1. Check application startup logs for MCP SDK initialization" -ForegroundColor Yellow
|
|
Write-Host "2. Review SDK documentation for transport configuration" -ForegroundColor Yellow
|
|
Write-Host "3. Consider testing with Claude Desktop (stdio transport)" -ForegroundColor Yellow
|
|
}
|
|
else {
|
|
Write-Host "1. Review detailed test results in JSON file" -ForegroundColor Yellow
|
|
Write-Host "2. Check server logs for any errors or warnings" -ForegroundColor Yellow
|
|
Write-Host "3. Proceed with Claude Desktop integration if stdio is supported" -ForegroundColor Yellow
|
|
}
|
|
|
|
Write-Host ""
|