15 connectors. One platform.

# Python
pip install opentelemetry-sdk opentelemetry-exporter-otlp \
  opentelemetry-instrumentation-openai

# Node.js
npm install @opentelemetry/sdk-node @opentelemetry/exporter-trace-otlp-grpc \
  @traceloop/instrumentation-openai

# Python — add at application startup
from opentelemetry.instrumentation.openai import OpenAIInstrumentor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor

provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(endpoint="http://localhost:4317"))
)

OpenAIInstrumentor().instrument(tracer_provider=provider)

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    send_batch_size: 1000
    timeout: 5s

exporters:
  otlp/qualityindex:
    endpoint: "https://ingest.qualityindex.ai:4317"
    headers:
      Authorization: "Bearer ${QI_API_KEY}"
    compression: gzip

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/qualityindex]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/qualityindex]

curl -s https://api.qualityindex.ai/api/v1/connectors/otel/health \
  -H "Authorization: Bearer ${QI_API_KEY}"

# Response
{
  "type": "otel",
  "status": "healthy",
  "last_span_received": "2026-03-07T14:32:00Z",
  "spans_ingested_24h": 1
}

# Permissions requested:
# - Contents (Read): Read code to analyze prompt files and configs
# - Pull requests (Read & Write): Create remediation PRs
# - Metadata (Read): Repository metadata
# - Webhooks: Receive push and PR events
#
# No access to: Actions, Issues, Discussions, Pages, Secrets

# Webhook events we listen to:
# - push (main/master): triggers rescore
# - pull_request (opened, synchronize): eval preview
# - installation_repositories (added, removed): updates repo list

# Each push generates a release snapshot:
{
  "tag": "v2.4.1",
  "commit_sha": "a1b2c3d",
  "quality_index": 87,
  "deployed_at": "2026-03-07T08:00:00Z"
}

# Remediation PR includes:
# - Which Eval Card detected the regression
# - Before/after scores with timestamps
# - Root cause analysis
# - The proposed fix
# - Link to the qualityindex.ai dashboard
#
# You review and merge — always human-in-the-loop.

# PR check status:
# ✅ qualityindex.ai — Quality Index: 89 (+2 from main)
# ❌ qualityindex.ai — Quality Index: 74 (-8 from main)
#    Reason: Faithfulness dropped below 0.80

# Configure in dashboard:
# Settings → Repositories → [repo] → Quality Gate
# Minimum Quality Index: 80
# Block on: Safety violations, Task Quality regression > 5 pts

# Install globally
npm install -g promptfoo

# Or as a dev dependency
npm install --save-dev promptfoo

# Run eval with JSON output
promptfoo eval --output results.json

# results.json contains:
# - Each test case with pass/fail per assertion
# - Scores, latency, token counts per provider
# - Aggregate pass rates per metric

# Upload results
curl -X POST https://api.qualityindex.ai/api/v1/connectors/promptfoo/import \
  -H "Authorization: Bearer ${QI_API_KEY}" \
  -H "Content-Type: application/json" \
  -d @results.json

# Response
{
  "imported": 248,
  "eval_cards_updated": ["ec_relevance", "ec_safety"],
  "quality_index_delta": "+3"
}

# .github/workflows/eval.yml
name: LLM Eval
on: [push]
jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: npm install -g promptfoo
      - run: promptfoo eval --output results.json
      - run: |
          curl -X POST https://api.qualityindex.ai/api/v1/connectors/promptfoo/import \
            -H "Authorization: Bearer ${{ secrets.QI_API_KEY }}" \
            -H "Content-Type: application/json" \
            -d @results.json

# Set as environment variable
export BRAINTRUST_API_KEY="bt_xxxxxxxxxxxxxxxxxxxx"

# We validate by fetching your project list:
GET https://api.braintrust.dev/v1/project
Headers: Authorization: Bearer bt_xxxx...

# Then list experiments in each project:
GET https://api.braintrust.dev/v1/experiment?project_id=<id>

# We import experiment scores:
GET https://api.braintrust.dev/v1/experiment/<id>/results

# Mapped fields:
# - scores → Eval Card rubric metrics
# - metadata.model → resource linkage
# - duration → Efficiency pillar
# - input/output tokens → cost calculation

POST /api/v1/eval-cards
{
  "name": "Braintrust Accuracy",
  "pillar": "task_quality",
  "rubric": [
    { "metric": "braintrust.accuracy", "threshold": 0.85, "weight": 0.6 },
    { "metric": "braintrust.relevance", "threshold": 0.80, "weight": 0.4 }
  ],
  "data_binding": {
    "source": "braintrust",
    "project_id": "proj_abc123",
    "experiment_filter": "latest"
  },
  "gate": false
}

# Phoenix uses standard OTel OTLP export.
# If your OTel Collector is configured for qualityindex.ai,
# Phoenix traces are already flowing.

# Verify:
curl -s https://api.qualityindex.ai/api/v1/connectors/otel/health \
  -H "Authorization: Bearer ${QI_API_KEY}"

# In the dashboard: Settings → Connectors → Add → Arize Phoenix
# Enter your Phoenix server URL and API key

# We fetch eval results:
GET https://your-phoenix-instance/api/v1/evaluations
Headers: Authorization: Bearer <phoenix_api_key>

# Imported: evaluation names, scores, labels, trace links

POST /api/v1/eval-cards
{
  "name": "Phoenix Retrieval Quality",
  "pillar": "task_quality",
  "rubric": [
    { "metric": "phoenix.relevance", "threshold": 0.80, "weight": 0.5 },
    { "metric": "phoenix.qa_correctness", "threshold": 0.85, "weight": 0.5 }
  ],
  "data_binding": {
    "source": "arize_phoenix",
    "evaluation_name": "retrieval_quality"
  },
  "gate": false
}

pip install guardrails-ai opentelemetry-sdk opentelemetry-exporter-otlp

# Initialize OTel (if not already done via the OTel connector)
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor

provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(endpoint="https://ingest.qualityindex.ai:4317"))
)

from opentelemetry import trace
from guardrails import Guard

tracer = trace.get_tracer("guardrails")

guard = Guard.from_pydantic(OutputModel)

with tracer.start_as_current_span("guardrails.validate") as span:
    result = guard(
        llm_api=openai.chat.completions.create,
        model="gpt-4o",
        messages=[{"role": "user", "content": user_input}]
    )
    span.set_attribute("guardrails.passed", result.validation_passed)
    span.set_attribute("guardrails.validators_run", len(result.validated_output or []))
    for i, log in enumerate(result.validation_logs):
        span.set_attribute(f"guardrails.validator.{i}.name", log.validator_name)
        span.set_attribute(f"guardrails.validator.{i}.passed", log.validation_passed)

POST /api/v1/eval-cards
{
  "name": "Output Validation Safety",
  "pillar": "safety",
  "rubric": [
    { "metric": "guardrails.pii_detection",    "threshold": 0.99, "weight": 0.4 },
    { "metric": "guardrails.hallucination",     "threshold": 0.95, "weight": 0.3 },
    { "metric": "guardrails.format_validation", "threshold": 0.98, "weight": 0.3 }
  ],
  "data_binding": { "source": "otel", "span_filter": "guardrails.validate" },
  "gate": true
}

# Install NeMo Guardrails with OTel support
pip install nemoguardrails opentelemetry-instrumentation-langchain

# If using LangChain, instrumenting LangChain also captures NeMo rails:
from opentelemetry.instrumentation.langchain import LangchainInstrumentor
LangchainInstrumentor().instrument()

# NeMo Guardrails config.yml
models:
  - type: main
    engine: openai
    model: gpt-4o

rails:
  input:
    flows:
      - self check input   # Jailbreak detection
      - check pii          # PII redaction
  output:
    flows:
      - self check output  # Hallucination prevention
      - check topic        # Off-topic redirect

# Each rail activation produces OTel span attributes:
# nemo.rail.name: "self check input"
# nemo.rail.action: "blocked" / "allowed" / "redirected"
# nemo.rail.confidence: 0.94

POST /api/v1/eval-cards
{
  "name": "NeMo Rail Safety",
  "pillar": "safety",
  "rubric": [
    { "metric": "nemo.jailbreak_block_rate",   "threshold": 0.98, "weight": 0.4 },
    { "metric": "nemo.pii_redaction_rate",      "threshold": 0.99, "weight": 0.3 },
    { "metric": "nemo.topic_compliance_rate",   "threshold": 0.95, "weight": 0.3 }
  ],
  "data_binding": { "source": "otel", "span_filter": "nemo.rail.*" },
  "gate": true
}

# In the dashboard: Settings → Connectors → Add → MLflow
# Enter your MLflow Tracking URI:
# - Self-hosted: http://mlflow.internal:5000
# - Databricks: https://<workspace>.cloud.databricks.com

# For Databricks, also provide a Personal Access Token.

# Under the hood:
POST http://mlflow.internal:5000/api/2.0/mlflow/runs/search
Body: {
  "experiment_ids": ["1", "2"],
  "filter_string": "metrics.accuracy > 0",
  "max_results": 100,
  "order_by": ["start_time DESC"]
}

# Imported fields: run_id, metrics.*, params.*, tags.*, artifacts

POST /api/v1/eval-cards
{
  "name": "MLflow Evaluation Metrics",
  "pillar": "task_quality",
  "rubric": [
    { "metric": "mlflow.accuracy",    "threshold": 0.90, "weight": 0.5 },
    { "metric": "mlflow.latency_p95", "threshold": 500,  "weight": 0.3 },
    { "metric": "mlflow.cost_per_req","threshold": 0.05, "weight": 0.2 }
  ],
  "data_binding": {
    "source": "mlflow",
    "experiment_id": "1",
    "run_filter": "latest"
  }
}