Apollo Observability

Overview

Comprehensive observability for Apollo.io integrations: Prometheus metrics (request count, latency, rate limits, credits), structured logging with PII redaction, OpenTelemetry tracing, and alerting rules. Tracks the metrics that matter: credit burn rate, enrichment success rate, and API health.

Prerequisites

• Valid Apollo API key
• Node.js 18+

Instructions

Step 1: Prometheus Metrics

// src/observability/metrics.ts
import { Counter, Histogram, Gauge, Registry } from 'prom-client';

export const registry = new Registry();

export const requestsTotal = new Counter({
  name: 'apollo_requests_total',
  help: 'Total Apollo API requests by endpoint and status',
  labelNames: ['endpoint', 'method', 'status'] as const,
  registers: [registry],
});

export const requestDuration = new Histogram({
  name: 'apollo_request_duration_seconds',
  help: 'Apollo API request duration',
  labelNames: ['endpoint'] as const,
  buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10],
  registers: [registry],
});

export const rateLimitRemaining = new Gauge({
  name: 'apollo_rate_limit_remaining',
  help: 'Remaining requests in current rate limit window',
  labelNames: ['endpoint'] as const,
  registers: [registry],
});

export const creditsUsed = new Counter({
  name: 'apollo_credits_used_total',
  help: 'Total Apollo enrichment credits consumed',
  labelNames: ['type'] as const,  // 'person', 'organization', 'bulk'
  registers: [registry],
});

export const enrichmentSuccessRate = new Gauge({
  name: 'apollo_enrichment_success_rate',
  help: 'Percentage of enrichment calls that found a match',
  registers: [registry],
});

Step 2: Axios Interceptors for Auto-Collection

// src/observability/instrument.ts
import { AxiosInstance } from 'axios';
import { requestsTotal, requestDuration, rateLimitRemaining, creditsUsed } from './metrics';

const CREDIT_ENDPOINTS = ['/people/match', '/people/bulk_match', '/organizations/enrich'];

export function instrumentClient(client: AxiosInstance) {
  client.interceptors.request.use((config) => {
    (config as any)._startTime = Date.now();
    return config;
  });

  client.interceptors.response.use(
    (response) => {
      const endpoint = response.config.url ?? 'unknown';
      const duration = (Date.now() - (response.config as any)._startTime) / 1000;

      requestsTotal.inc({ endpoint, method: response.config.method?.toUpperCase() ?? 'GET', status: String(response.status) });
      requestDuration.observe({ endpoint }, duration);

      // Rate limit tracking
      const remaining = response.headers['x-rate-limit-remaining'];
      if (remaining) rateLimitRemaining.set({ endpoint }, parseInt(remaining, 10));

      // Credit tracking
      if (CREDIT_ENDPOINTS.some((ep) => endpoint.includes(ep))) {
        const type = endpoint.includes('bulk') ? 'bulk' : endpoint.includes('organization') ? 'organization' : 'person';
        const count = response.data?.matches?.length ?? 1;
        creditsUsed.inc({ type }, count);
      }

      return response;
    },
    (err) => {
      requestsTotal.inc({
        endpoint: err.config?.url ?? 'unknown',
        method: err.config?.method?.toUpperCase() ?? 'GET',
        status: String(err.response?.status ?? 0),
      });
      return Promise.reject(err);
    },
  );
}

Step 3: Structured Logging with PII Redaction

// src/observability/logger.ts
import pino from 'pino';

export const logger = pino({
  level: process.env.LOG_LEVEL ?? 'info',
  redact: {
    paths: ['*.email', '*.phone_numbers', '*.linkedin_url', 'headers.x-api-key'],
    censor: '[REDACTED]',
  },
  formatters: { level: (label) => ({ level: label }) },
  transport: process.env.NODE_ENV !== 'production' ? { target: 'pino-pretty' } : undefined,
});

export const apolloLog = logger.child({ service: 'apollo' });

// Usage:
// apolloLog.info({ endpoint: '/mixed_people/api_search', results: 25 }, 'Search completed');
// apolloLog.warn({ endpoint: '/people/match', status: 429 }, 'Rate limited');
// apolloLog.error({ err, endpoint: '/contacts' }, 'Request failed');

Step 4: OpenTelemetry Tracing

// src/observability/tracing.ts
import { trace, SpanStatusCode } from '@opentelemetry/api';
import { AxiosInstance } from 'axios';

const tracer = trace.getTracer('apollo-integration');

export function addTracing(client: AxiosInstance) {
  client.interceptors.request.use((config) => {
    const span = tracer.startSpan(`apollo.${config.method?.toUpperCase()} ${config.url}`);
    span.setAttribute('apollo.endpoint', config.url ?? '');
    (config as any)._span = span;
    return config;
  });

  client.interceptors.response.use(
    (response) => {
      const span = (response.config as any)._span;
      if (span) {
        span.setAttribute('http.status_code', response.status);
        span.setAttribute('apollo.rate_limit_remaining', response.headers['x-rate-limit-remaining'] ?? 'unknown');
        span.setStatus({ code: SpanStatusCode.OK });
        span.end();
      }
      return response;
    },
    (err) => {
      const span = (err.config as any)?._span;
      if (span) {
        span.setAttribute('http.status_code', err.response?.status ?? 0);
        span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
        span.end();
      }
      return Promise.reject(err);
    },
  );
}

Step 5: Alerting Rules

# prometheus/apollo-alerts.yml
groups:
  - name: apollo-integration
    rules:
      - alert: ApolloHighErrorRate
        expr: rate(apollo_requests_total{status=~"4..|5.."}[5m]) / rate(apollo_requests_total[5m]) > 0.1
        for: 5m
        labels: { severity: critical }
        annotations: { summary: "Apollo API error rate > 10% for 5 minutes" }

      - alert: ApolloRateLimitLow
        expr: apollo_rate_limit_remaining < 20
        for: 1m
        labels: { severity: warning }
        annotations: { summary: "Apollo rate limit below 20 remaining requests" }

      - alert: ApolloHighLatency
        expr: histogram_quantile(0.95, rate(apollo_request_duration_seconds_bucket[5m])) > 5
        for: 10m
        labels: { severity: warning }
        annotations: { summary: "Apollo p95 latency > 5s for 10 minutes" }

      - alert: ApolloCreditBurnRate
        expr: rate(apollo_credits_used_total[1h]) * 24 > 500
        for: 30m
        labels: { severity: warning }
        annotations: { summary: "Apollo credit burn rate projects > 500/day" }

Step 6: Metrics Endpoint

import express from 'express';
import { registry } from './metrics';

const metricsApp = express();
metricsApp.get('/metrics', async (_, res) => {
  res.set('Content-Type', registry.contentType);
  res.end(await registry.metrics());
});
metricsApp.get('/health', (_, res) => res.json({ status: 'ok' }));
// Keep metrics private; expose it to a scraper through a local agent or an
// explicitly authenticated, network-restricted deployment path.
metricsApp.listen(9090, '127.0.0.1', () => console.log('Metrics on 127.0.0.1:9090'));

Output

• Prometheus metrics: requests, duration, rate limits, credits, enrichment success
• Axios interceptors for automatic collection on every API call
• Pino structured logger with PII redaction
• OpenTelemetry tracing spans for distributed tracing
• Alerting rules for errors, rate limits, latency, and credit burn rate
• /metrics and /health HTTP endpoints

Examples

For a new enrichment worker, instrument the client before its first request, scrape /metrics only through the approved local collector or a network-restricted authenticated path, and confirm that a synthetic 429 increments the error and rate-limit signals without emitting a contact record or API key. Temporarily lower the rate-limit threshold in a non-production environment to prove the alert reaches the on-call route, then restore the production threshold. If a metric endpoint is publicly reachable, labels carry unbounded values, or the alert has no owner, block the rollout until the exposure or operational gap is corrected.

Error Handling

Issue	Resolution
Missing metrics	Verify instrumentClient() called before first API call
Alert noise	Tune for duration and thresholds
Log volume	Use LOG_LEVEL=warn in production
Credit burn alert	Review enrichment scoring thresholds in apollo-cost-tuning

Resources

• Prometheus Node.js Client
• OpenTelemetry JavaScript
• Pino Logger
• Apollo API Usage Stats

Next Steps

Proceed to apollo-incident-runbook for incident response.