SDK Examples for Pagination and Error Handling

This page collects the most reusable integration patterns across languages:

fetching every page of a result set
respecting rate limits
retrying transient failures
separating retryable and non-retryable errors

Use these examples when building internal clients, cron jobs, ETL pipelines, or customer-facing integrations.

Pagination

Python (`requests`)

import time
import requests

BASE_URL = "https://api.oilpriceapi.com/v1"

def get_all_pages(endpoint, api_key, params=None, per_page=100):
    params = dict(params or {})
    params["page"] = 1
    params["per_page"] = per_page
    items = []

    while True:
        response = requests.get(
            f"{BASE_URL}{endpoint}",
            params=params,
            headers={"Authorization": f"Token {api_key}"},
            timeout=30,
        )

        if response.status_code == 429:
            retry_after = int(response.headers.get("Retry-After", "60"))
            time.sleep(retry_after)
            continue

        response.raise_for_status()
        payload = response.json()
        page_items = payload.get("data", [])

        if isinstance(page_items, list):
            items.extend(page_items)
        elif page_items:
            items.append(page_items)

        current_page = int(response.headers.get("X-Page", params["page"]))
        total_pages = int(response.headers.get("X-Total-Pages", current_page))
        if current_page >= total_pages:
            break

        params["page"] = current_page + 1

    return items

JavaScript (`fetch`)

async function getAllPages(endpoint, apiKey, params = {}) {
  const items = [];
  let page = 1;

  while (true) {
    const url = new URL(`https://api.oilpriceapi.com/v1${endpoint}`);
    url.searchParams.set("page", String(page));
    url.searchParams.set("per_page", "100");

    for (const [key, value] of Object.entries(params)) {
      url.searchParams.set(key, String(value));
    }

    const response = await fetch(url, {
      headers: { Authorization: `Token ${apiKey}` },
    });

    if (response.status === 429) {
      const retryAfter = Number(response.headers.get("Retry-After") || "60");
      await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000));
      continue;
    }

    if (!response.ok) {
      throw new Error(`Request failed with ${response.status}`);
    }

    const payload = await response.json();
    const pageItems = payload.data;

    if (Array.isArray(pageItems)) {
      items.push(...pageItems);
    } else if (pageItems) {
      items.push(pageItems);
    }

    const currentPage = Number(response.headers.get("X-Page") || page);
    const totalPages = Number(
      response.headers.get("X-Total-Pages") || currentPage,
    );

    if (currentPage >= totalPages) {
      return items;
    }

    page = currentPage + 1;
  }
}

Ruby (`Net::HTTP`)

require "json"
require "net/http"
require "uri"

def get_all_pages(endpoint, api_key, params = {})
  items = []
  page = 1

  loop do
    uri = URI("https://api.oilpriceapi.com/v1#{endpoint}")
    uri.query = URI.encode_www_form(params.merge(page: page, per_page: 100))

    request = Net::HTTP::Get.new(uri)
    request["Authorization"] = "Token #{api_key}"

    response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
      http.request(request)
    end

    if response.code.to_i == 429
      sleep((response["Retry-After"] || "60").to_i)
      next
    end

    raise "Request failed with #{response.code}" unless response.code.to_i.between?(200, 299)

    payload = JSON.parse(response.body)
    page_items = payload["data"]

    if page_items.is_a?(Array)
      items.concat(page_items)
    elsif page_items
      items << page_items
    end

    current_page = (response["X-Page"] || page).to_i
    total_pages = (response["X-Total-Pages"] || current_page).to_i
    break if current_page >= total_pages

    page = current_page + 1
  end

  items
end

PHP (`cURL`)

<?php

function getAllPages(string $endpoint, string $apiKey, array $params = []): array
{
    $items = [];
    $page = 1;

    while (true) {
        $query = http_build_query(array_merge($params, [
            'page' => $page,
            'per_page' => 100,
        ]));

        $ch = curl_init("https://api.oilpriceapi.com/v1{$endpoint}?{$query}");
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_HEADER => true,
            CURLOPT_HTTPHEADER => [
                "Authorization: Token {$apiKey}",
            ],
            CURLOPT_TIMEOUT => 30,
        ]);

        $raw = curl_exec($ch);
        $status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
        $headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
        $headers = substr($raw, 0, $headerSize);
        $body = substr($raw, $headerSize);

        if ($status === 429) {
            preg_match('/Retry-After:\\s*(\\d+)/i', $headers, $matches);
            sleep((int)($matches[1] ?? 60));
            curl_close($ch);
            continue;
        }

        if ($status < 200 || $status >= 300) {
            curl_close($ch);
            throw new RuntimeException("Request failed with {$status}");
        }

        $payload = json_decode($body, true, 512, JSON_THROW_ON_ERROR);
        $pageItems = $payload['data'] ?? [];

        if (array_is_list($pageItems)) {
            $items = array_merge($items, $pageItems);
        } elseif (!empty($pageItems)) {
            $items[] = $pageItems;
        }

        preg_match('/X-Page:\\s*(\\d+)/i', $headers, $pageMatch);
        preg_match('/X-Total-Pages:\\s*(\\d+)/i', $headers, $totalMatch);
        $currentPage = (int)($pageMatch[1] ?? $page);
        $totalPages = (int)($totalMatch[1] ?? $currentPage);

        curl_close($ch);

        if ($currentPage >= $totalPages) {
            return $items;
        }

        $page = $currentPage + 1;
    }
}

Error Handling and Retries

Retry only failures that are likely to succeed on a later attempt.

Retryable Cases

429 Too Many Requests
408 Request Timeout
5xx server failures
temporary network errors

Non-Retryable Cases

400 invalid parameters
401 invalid API key
403 insufficient plan
404 invalid endpoint or missing resource

JavaScript Backoff Example

async function fetchWithRetry(url, options = {}, maxRetries = 4) {
  for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
    try {
      const response = await fetch(url, options);

      if (response.ok) {
        return response;
      }

      if (![408, 429, 500, 502, 503, 504].includes(response.status)) {
        throw new Error(`Non-retryable status ${response.status}`);
      }

      const retryAfter = Number(response.headers.get("Retry-After") || "0");
      const backoffMs =
        retryAfter > 0
          ? retryAfter * 1000
          : Math.min(1000 * 2 ** attempt, 10000);

      await new Promise((resolve) => setTimeout(resolve, backoffMs));
    } catch (error) {
      if (attempt === maxRetries) {
        throw error;
      }

      const backoffMs = Math.min(1000 * 2 ** attempt, 10000);
      await new Promise((resolve) => setTimeout(resolve, backoffMs));
    }
  }

  throw new Error("Exhausted retries");
}

Python Backoff Example

import time
import requests

RETRYABLE_STATUS_CODES = {408, 429, 500, 502, 503, 504}

def fetch_with_retry(url, api_key, max_retries=4):
    headers = {"Authorization": f"Token {api_key}"}

    for attempt in range(max_retries + 1):
        try:
            response = requests.get(url, headers=headers, timeout=30)

            if response.ok:
                return response

            if response.status_code not in RETRYABLE_STATUS_CODES:
                response.raise_for_status()

            retry_after = int(response.headers.get("Retry-After", "0"))
            delay = retry_after if retry_after > 0 else min(2 ** attempt, 10)
            time.sleep(delay)
        except requests.RequestException:
            if attempt == max_retries:
                raise
            time.sleep(min(2 ** attempt, 10))

    raise RuntimeError("Exhausted retries")