crawlee-http.api

Public API Report File for "@crawlee/http"

Do not edit this file. It is a report generated by API Extractor.

import type { Awaitable } from '@crawlee/types';
import { BasicCrawler } from '@crawlee/basic';
import { BasicCrawlerOptions } from '@crawlee/basic';
import type { CheerioAPI } from 'cheerio';
import { ConcurrencySystem } from '@crawlee/basic';
import type { ConcurrencySystemOptions } from '@crawlee/basic';
import { ContextPipeline } from '@crawlee/basic';
import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/core';
import type { CrawlingContext } from '@crawlee/basic';
import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';
import type { Dictionary } from '@crawlee/types';
import { ErrorHandler } from '@crawlee/basic';
import { GetUserDataFromRequest } from '@crawlee/basic';
import type { JsonValue } from 'type-fest';
import { LoadedRequest } from '@crawlee/core';
import { Request as Request_2 } from '@crawlee/basic';
import type { Request as Request_3 } from '@crawlee/core';
import { RequestHandler } from '@crawlee/basic';
import type { RequireContextPipeline } from '@crawlee/basic';
import { RouterHandler } from '@crawlee/basic';
import { RouterRoutes } from '@crawlee/basic';
import { RouteSchemas } from '@crawlee/basic';
import { RoutesFromSchemas } from '@crawlee/basic';
import { Transform } from 'node:stream';

// @public
export function ByteCounterStream(input: {
    logTransferredBytes: (transferredBytes: number) => void;
    loggingInterval?: number;
}): Transform;

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
    request: LoadedRequest<Request_2<UserData>>;
    response: Response;
}

// @public
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

// @public (undocumented)
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

// @public (undocumented)
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

// @public
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

// @public (undocumented)
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

// @public (undocumented)
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

// @public
export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
    constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
    // (undocumented)
    protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2, FileDownloadCrawlingContext>;
}

// @public (undocumented)
export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext_2<UserData> {
    // (undocumented)
    contentType: {
        type: string;
        encoding: BufferEncoding;
    };
    // (undocumented)
    request: LoadedRequest<Request_3<UserData>>;
    // (undocumented)
    response: Response;
}

// @public (undocumented)
export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext_2, FileDownloadCrawlingContext<UserData> & ContextExtension>;

// @public (undocumented)
export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;

// @public (undocumented)
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;

// @public
export const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySystemOptions;

// @public
export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
    // (undocumented)
    protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
    protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
    // (undocumented)
    protected getNavigationTimeoutMillis(): number;
    // (undocumented)
    protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
}

// @public (undocumented)
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
    additionalMimeTypes?: string[];
    forceResponseEncoding?: string;
    ignoreTlsErrors?: boolean;
    navigationTimeoutSecs?: number;
    postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse & ContextExtension) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
    preNavigationHooks?: InternalHttpHook<CrawlingContext<any>, ContextExtension>[];
    saveResponseCookies?: boolean;
    suggestResponseEncoding?: string;
}

// @public (undocumented)
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
}

// @public (undocumented)
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;

// @public (undocumented)
export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;

// @public (undocumented)
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;

// @public (undocumented)
export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
    body: string | Buffer;
    contentType: {
        type: string;
        encoding: BufferEncoding;
    };
    json: JSONData;
    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
    waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
}

// @public (undocumented)
export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;

// @public
export function MinimumSpeedStream(input: {
    minSpeedKbps: number;
    historyLengthMs?: number;
    checkProgressInterval?: number;
}): Transform;


export * from "@crawlee/basic";

// (No @packageDocumentation comment for this package)