crawlee-basic.api

Public API Report File for "@crawlee/basic"

Do not edit this file. It is a report generated by API Extractor.

import type { AddRequestsBatchedOptions } from '@crawlee/core';
import type { AddRequestsBatchedResult } from '@crawlee/core';
import type { Awaitable } from '@crawlee/types';
import { BaseHttpClient } from '@crawlee/http-client';
import { ConcurrencySystem } from '@crawlee/core';
import { ConcurrencySystemOptions } from '@crawlee/core';
import { Configuration } from '@crawlee/core';
import { ContextPipeline } from '@crawlee/core';
import type { CrawleeLogger } from '@crawlee/core';
import { CrawlingContext } from '@crawlee/core';
import { Dataset } from '@crawlee/core';
import type { DatasetExportOptions } from '@crawlee/core';
import type { Dictionary } from '@crawlee/types';
import type { EnqueueUrlsOptions } from '@crawlee/core';
import { EventManager } from '@crawlee/core';
import type { FinalStatistics } from '@crawlee/core';
import type { GetUserDataFromRequest } from '@crawlee/core';
import { IConcurrencySystem } from '@crawlee/core';
import { IProxyConfiguration } from '@crawlee/core';
import { IRequestLoader } from '@crawlee/core';
import { IRequestManager } from '@crawlee/core';
import type { ISession } from '@crawlee/types';
import type { ISessionPool } from '@crawlee/types';
import { IStatistics } from '@crawlee/core';
import type { ProxyInfo } from '@crawlee/types';
import type { ReadonlyDeep } from 'type-fest';
import { Request as Request_2 } from '@crawlee/core';
import { RequestQueue } from '@crawlee/core';
import { RobotsTxtFile } from '@crawlee/utils';
import type { RouterHandler } from '@crawlee/core';
import type { RouterRoutes } from '@crawlee/core';
import type { SetStatusMessageOptions } from '@crawlee/types';
import type { SkippedRequestCallback } from '@crawlee/core';
import type { StatisticState } from '@crawlee/core';
import type { StorageBackend } from '@crawlee/types';
import type { StorageIdentifier } from '@crawlee/core';
import { StorageWritePolicy } from '@crawlee/core';
import type { TaskLoopPredicates } from '@crawlee/core';
import { TimeoutError } from '@apify/timeout';
import type { TypedRequestsLike } from '@crawlee/core';

// @public (undocumented)
export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
    constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<CrawlingContext, Context>);
    // (undocumented)
    protected readonly additionalHttpErrorStatusCodes: Set<number>;
    addRequests(requests: ReadonlyDeep<TypedRequestsLike<Routes>>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
    get basicContextPipeline(): ContextPipeline<{
        request: Request_2;
    }, CrawlingContext>;
    // (undocumented)
    protected blockedStatusCodes: Set<number>;
    protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
    get concurrencySystem(): IConcurrencySystem | undefined;
    // (undocumented)
    get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
    protected createDefaultConcurrencySystem(options: ConcurrencySystemOptions): ConcurrencySystem;
    exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;
    // (undocumented)
    protected getCookieHeaderFromRequest(request: Request_2): string;
    getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;
    getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
    protected getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
    protected getNavigationTimeoutMillis(): number;
    getRequestManager(): Promise<IRequestManager>;
    // @deprecated (undocumented)
    getRequestQueue(): Promise<IRequestManager>;
    // (undocumented)
    protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
    // (undocumented)
    hasFinishedBefore: boolean;
    // (undocumented)
    protected readonly httpClient: BaseHttpClient;
    protected init(): Promise<void>;
    // (undocumented)
    protected readonly internalTimeoutMillis: number;
    protected isErrorStatusCode(status: number): boolean;
    protected isProxyError(error: Error): boolean;
    // (undocumented)
    get log(): CrawleeLogger;
    pause(timeoutSecs?: number): Promise<void>;
    readonly proxyConfiguration?: IProxyConfiguration;
    pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
    protected recordDomainRateLimit(url: string, retryAfterHeader?: string | null): boolean;
    // (undocumented)
    protected readonly requestHandler: RequestHandler<ExtendedContext>;
    protected requestManager?: IRequestManager;
    resume(): void;
    // (undocumented)
    protected readonly retryOnBlocked: boolean;
    readonly router: RouterHandler<Context, Routes>;
    run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
    // (undocumented)
    running: boolean;
    // (undocumented)
    protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
    get sessionPool(): ISessionPool;
    setStatusMessage(message: string, options?: SetStatusMessageOptions): void;
    get statistics(): IStatistics<StatisticStateExtension>;
    stop(reason?: string): void;
    teardown(): Promise<void>;
    protected throwOnBlockedRequest(statusCode: number): void;
    // (undocumented)
    useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
}

// @public (undocumented)
export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> {
    additionalHttpErrorStatusCodes?: number[];
    blockedStatusCodes?: number[];
    concurrencySystem?: IConcurrencySystem;
    configuration?: Configuration;
    contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
    errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
    eventManager?: EventManager;
    extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
    failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
    httpClient?: BaseHttpClient;
    id?: string;
    ignoreHttpErrorStatusCodes?: number[];
    keepAlive?: boolean;
    logger?: CrawleeLogger;
    maxConcurrency?: number;
    maxCrawlDepth?: number;
    maxRequestRetries?: number;
    maxRequestsPerCrawl?: number;
    maxRequestsPerMinute?: number;
    minConcurrency?: number;
    onSkippedRequest?: SkippedRequestCallback;
    proxyConfiguration?: IProxyConfiguration;
    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
    requestHandlerTimeoutSecs?: number;
    // @deprecated
    requestList?: IRequestLoader;
    requestManager?: IRequestManager;
    // @deprecated
    requestQueue?: RequestQueue;
    respectRobotsTxtFile?: boolean | {
        userAgent?: string;
    };
    retryOnBlocked?: boolean;
    sameDomainDelaySecs?: number;
    sessionPool?: ISessionPool;
    statistics?: IStatistics<StatisticStateExtension>;
    statusMessageCallback?: StatusMessageCallback;
    statusMessageLoggingInterval?: number;
    storageBackend?: StorageBackend;
    taskLoopOptions?: TaskLoopPredicates;
    transactionalStorage?: boolean | Partial<StorageWritePolicy>;
}

// @public (undocumented)
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
}

// @public (undocumented)
export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, EnqueueUrlsOptions {
}

// @public (undocumented)
export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
}

// @public (undocumented)
export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
    purgeRequestQueue?: boolean;
}

// @public
export function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

// @public (undocumented)
export function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

// @public (undocumented)
export interface CreateContextOptions {
    // (undocumented)
    proxyInfo?: ProxyInfo;
    // (undocumented)
    request: Request_2;
    // (undocumented)
    session: ISession;
}

// @public
export type ErrorHandler<BaseContext extends CrawlingContext = CrawlingContext, ExtendedContext extends BaseContext = BaseContext> = (inputs: BaseContext & Partial<ExtendedContext>, error: Error) => Awaitable<void>;

// @public (undocumented)
export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;

// @public (undocumented)
export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
    contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
};

// @public (undocumented)
export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;

// @public (undocumented)
export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any, any, any, any> = BasicCrawler<Context>> {
    // (undocumented)
    crawler: Crawler;
    // (undocumented)
    message: string;
    // (undocumented)
    previousState: StatisticState;
    // (undocumented)
    state: StatisticState;
}


export * from "@crawlee/core";

// (No @packageDocumentation comment for this package)