Retrieve large datasets

curl -X POST "https://v3-api.newscatcherapi.com/api/aggregation_count" \
  -H "x-api-token: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "q": "artificial intelligence",
    "lang": "en",
    "aggregation_by": "day"
  }'

from newscatcher import NewscatcherApi

client = NewscatcherApi(api_key="YOUR_API_KEY")

result = client.aggregation.post(
    q="artificial intelligence",
    lang="en",
    aggregation_by="day",
)

print(f"Total articles: {result.total_hits}")
for bucket in result.aggregations[0].aggregation_count:
    print(f"{bucket.time_frame[:10]}: {bucket.article_count:,} articles")

import { NewscatcherApiClient } from "newscatcher-sdk";

const client = new NewscatcherApiClient({ apiKey: "YOUR_API_KEY" });

const result = await client.aggregation.post({
  q: "artificial intelligence",
  lang: "en",
  aggregationBy: "day",
});

console.log(`Total articles: ${result.totalHits}`);
result.aggregations?.[0].aggregationCount?.forEach((bucket) => {
  console.log(`${bucket.timeFrame?.slice(0, 10)}: ${bucket.articleCount}`);
});

import com.newscatcher.api.NewscatcherApiClient;
import com.newscatcher.api.resources.aggregation.requests.PostAggregationRequest;

NewscatcherApiClient client = NewscatcherApiClient.builder()
    .apiKey("YOUR_API_KEY")
    .build();

var result = client.aggregation().post(
    PostAggregationRequest.builder()
        .q("artificial intelligence")
        .lang("en")
        .aggregationBy("day")
        .build()
);

System.out.println("Total articles: " + result.getTotalHits());
result.getAggregations().get(0).getAggregationCount().forEach(bucket ->
    System.out.println(bucket.getTimeFrame() + ": " + bucket.getArticleCount())
);

import time
from newscatcher import NewscatcherApi
from newscatcher.core import ApiError

client = NewscatcherApi(api_key="YOUR_API_KEY")

def get_all_articles(query: str, from_date: str, to_date: str, chunk_hours: int) -> list:
    from datetime import datetime, timedelta

    articles = []
    window_start = datetime.fromisoformat(from_date)
    end = datetime.fromisoformat(to_date)

    while window_start < end:
        window_end = min(window_start + timedelta(hours=chunk_hours), end)

        page = 1
        total_pages = 1

        while page <= total_pages:
            try:
                response = client.search.post(
                    q=query,
                    lang="en",
                    from_=window_start.isoformat(),
                    to=window_end.isoformat(),
                    page=page,
                    page_size=1000,
                )
                articles.extend(response.articles)
                total_pages = response.total_pages
                page += 1
                time.sleep(0.5)
            except ApiError as e:
                print(f"Error on page {page}: {e}")
                break

        window_start = window_end

    return articles

articles = get_all_articles(
    "artificial intelligence",
    "2026-05-04T00:00:00",
    "2026-05-11T00:00:00",
    chunk_hours=6,
)
print(f"Retrieved {len(articles)} articles")

import { NewscatcherApiClient } from "newscatcher-sdk";

const client = new NewscatcherApiClient({ apiKey: "YOUR_API_KEY" });

async function getAllArticles(
  query: string,
  from: Date,
  to: Date,
  chunkHours: number
): Promise<object[]> {
  const articles: object[] = [];
  let windowStart = new Date(from);

  while (windowStart < to) {
    const windowEnd = new Date(
      Math.min(windowStart.getTime() + chunkHours * 3600 * 1000, to.getTime())
    );

    let page = 1;
    let totalPages = 1;

    while (page <= totalPages) {
      const response = await client.search.post({
        q: query,
        lang: "en",
        from: windowStart,
        to: windowEnd,
        pageSize: 1000,
        page,
      });

      articles.push(...(response.articles ?? []));
      totalPages = response.totalPages ?? 1;
      page++;
    }

    windowStart = windowEnd;
  }

  return articles;
}

const articles = await getAllArticles(
  "artificial intelligence",
  new Date("2026-05-04T00:00:00"),
  new Date("2026-05-11T00:00:00"),
  6,
);
console.log(`Retrieved ${articles.length} articles`);

import com.newscatcher.api.NewscatcherApiClient;
import com.newscatcher.api.resources.search.requests.PostSearchRequest;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;

NewscatcherApiClient client = NewscatcherApiClient.builder()
    .apiKey("YOUR_API_KEY")
    .build();

List<Object> getAllArticles(String query, Instant from, Instant to, int chunkHours) {
    List<Object> articles = new ArrayList<>();
    Instant windowStart = from;

    while (windowStart.isBefore(to)) {
        Instant windowEnd = windowStart.plus(chunkHours, ChronoUnit.HOURS);
        if (windowEnd.isAfter(to)) windowEnd = to;

        int page = 1;
        int totalPages = 1;

        while (page <= totalPages) {
            var response = client.search().post(
                PostSearchRequest.builder()
                    .q(query)
                    .lang("en")
                    .from(windowStart.toString())
                    .to(windowEnd.toString())
                    .pageSize(1000)
                    .page(page)
                    .build()
            );

            articles.addAll(response.getArticles());
            totalPages = response.getTotalPages();
            page++;
        }

        windowStart = windowEnd;
    }

    return articles;
}