Skip to main content
Version: 4.0 (RC)

crawlee-types.api

Public API Report File for "@crawlee/types"​

Do not edit this file. It is a report generated by API Extractor.


import type { Readable } from 'node:stream';

// @public (undocumented)
export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';

// @public
export interface BaseHttpClient {
    sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
}

// @public (undocumented)
export interface BatchAddRequestsResult {
    // (undocumented)
    processedRequests: ProcessedRequest[];
    // (undocumented)
    unprocessedRequests: UnprocessedRequest[];
}

// @public (undocumented)
export interface BrowserLikeResponse {
    // (undocumented)
    headers(): Dictionary<string | string[]>;
    // (undocumented)
    url(): string;
}

// @public (undocumented)
export interface Cookie {
    domain?: string;
    expires?: number;
    httpOnly?: boolean;
    name: string;
    path?: string;
    priority?: 'Low' | 'Medium' | 'High';
    sameParty?: boolean;
    sameSite?: 'Strict' | 'Lax' | 'None';
    secure?: boolean;
    sourcePort?: number;
    sourceScheme?: 'Unset' | 'NonSecure' | 'Secure';
    url?: string;
    value: string;
}

// @public
export interface CookieJar {
    // (undocumented)
    clone(): Promise<CookieJar>;
    // (undocumented)
    getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
    // (undocumented)
    getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
    // (undocumented)
    getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
    // (undocumented)
    serialize(): Promise<SerializedCookieJar>;
    // (undocumented)
    setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
    // (undocumented)
    toJSON(): SerializedCookieJar | undefined;
}

// @public
export interface CookieJarGetCookiesOptions {
    // (undocumented)
    allPaths?: boolean;
    // (undocumented)
    expire?: boolean;
    // (undocumented)
    http?: boolean;
    // (undocumented)
    sameSiteContext?: 'none' | 'lax' | 'strict';
    // (undocumented)
    sort?: boolean;
}

// @public
export interface CookieJarSetCookieOptions {
    // (undocumented)
    http?: boolean;
    // (undocumented)
    ignoreError?: boolean;
    // (undocumented)
    loose?: boolean;
    // (undocumented)
    now?: Date;
    // (undocumented)
    sameSiteContext?: 'strict' | 'lax' | 'none';
}

// @public
export interface CrawleeLogger {
    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    debug(message: string, data?: Record<string, unknown>): void;
    deprecated(message: string): void;
    error(message: string, data?: Record<string, unknown>): void;
    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
    getOptions(): CrawleeLoggerOptions;
    info(message: string, data?: Record<string, unknown>): void;
    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
    perf(message: string, data?: Record<string, unknown>): void;
    setOptions(options: Partial<CrawleeLoggerOptions>): void;
    softFail(message: string, data?: Record<string, unknown>): void;
    warning(message: string, data?: Record<string, unknown>): void;
    warningOnce(message: string): void;
}

// @public
export interface CrawleeLoggerOptions {
    prefix?: string | null;
}

// @public (undocumented)
export interface DatasetBackend<Data extends Dictionary = Dictionary> {
    drop(): Promise<void>;
    getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Data>>;
    getMetadata(): Promise<DatasetInfo>;
    purge(): Promise<void>;
    pushData(items: Data[]): Promise<void>;
}

// @public (undocumented)
export interface DatasetBackendListOptions {
    // (undocumented)
    desc?: boolean;
    // (undocumented)
    limit?: number;
    // (undocumented)
    offset?: number;
}

// @public (undocumented)
export interface DatasetInfo {
    // (undocumented)
    accessedAt: Date;
    // (undocumented)
    createdAt: Date;
    // (undocumented)
    id: string;
    // (undocumented)
    itemCount: number;
    // (undocumented)
    modifiedAt: Date;
    // (undocumented)
    name?: string;
}

// @public (undocumented)
export type Dictionary<T = any> = Record<PropertyKey, T>;

// @public
export interface HttpRequest {
    // (undocumented)
    body?: Readable;
    // (undocumented)
    cookieJar?: CookieJar;
    // (undocumented)
    encoding?: BufferEncoding;
    // (undocumented)
    followRedirect?: boolean | ((response: any) => boolean);
    // (undocumented)
    headerGenerator?: {
        getHeaders: (options: Record<string, unknown>) => Record<string, string>;
    };
    // (undocumented)
    headerGeneratorOptions?: Record<string, unknown>;
    // (undocumented)
    headers?: Headers;
    // (undocumented)
    insecureHTTPParser?: boolean;
    // (undocumented)
    maxRedirects?: number;
    // (undocumented)
    method?: AllowedHttpMethods;
    // (undocumented)
    proxyUrl?: string;
    // (undocumented)
    sessionToken?: object;
    // (undocumented)
    signal?: AbortSignal;
    // (undocumented)
    throwHttpErrors?: boolean;
    // (undocumented)
    timeout?: number;
    // (undocumented)
    url: string | URL;
    // (undocumented)
    useHeaderGenerator?: boolean;
}

// @public
export interface HttpRequestOptions extends HttpRequest {
    form?: Record<string, string>;
    json?: unknown;
    password?: string;
    searchParams?: SearchParams;
    username?: string;
}

// @public
export interface IBrowserPool<Page = unknown> {
    closePage(page: Page, options?: {
        error?: Error;
    }): Promise<void>;
    extractPageState(page: Page): Promise<PageState>;
    injectPageState(page: Page, state: PageState): Promise<void>;
    newPage(options?: NewPageOptions): Promise<Page>;
}

// @public
export interface ISession {
    // (undocumented)
    cookieJar: CookieJar;
    // (undocumented)
    fingerprint?: SessionFingerprint;
    // (undocumented)
    readonly id: string;
    isUsable(): boolean;
    markBad(): void;
    markGood(): void;
    // (undocumented)
    proxyInfo?: ProxyInfo;
    retire(): void;
}

// @public
export interface ISessionPool {
    getSession(sessionId?: string): Promise<ISession | undefined>;
}

// @public
export interface KeyValueStoreBackend {
    deleteValue(key: string): Promise<void>;
    drop(): Promise<void>;
    getMetadata(): Promise<KeyValueStoreInfo>;
    getPublicUrl(key: string): Promise<string | undefined>;
    getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
    listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
    purge(): Promise<void>;
    recordExists(key: string): Promise<boolean>;
    setValue(record: KeyValueStoreInputRecord): Promise<void>;
}

// @public (undocumented)
export interface KeyValueStoreInfo {
    // (undocumented)
    accessedAt: Date;
    // (undocumented)
    createdAt: Date;
    // (undocumented)
    id: string;
    // (undocumented)
    modifiedAt: Date;
    // (undocumented)
    name?: string;
}

// @public
export interface KeyValueStoreInputRecord {
    // (undocumented)
    contentType?: string;
    // (undocumented)
    key: string;
    // (undocumented)
    value: KeyValueStoreRecordInputValue;
}

// @public (undocumented)
export interface KeyValueStoreItemData {
    contentType: string;
    // (undocumented)
    key: string;
    // (undocumented)
    size: number;
}

// @public (undocumented)
export interface KeyValueStoreListKeysOptions {
    exclusiveStartKey?: string;
    limit?: number;
    prefix?: string;
}

// @public
export interface KeyValueStoreListKeysResult {
    count: number;
    exclusiveStartKey?: string;
    isTruncated: boolean;
    items: KeyValueStoreItemData[];
    limit: number;
    nextExclusiveStartKey?: string;
}

// @public
export interface KeyValueStoreRecord {
    // (undocumented)
    contentType?: string;
    // (undocumented)
    key: string;
    // (undocumented)
    value: Buffer | ArrayBuffer;
}

// @public
export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;

// @public
export interface NewPageOptions {
    id?: string;
    session?: ISession;
}

// @public
export interface PageState {
    cookies: Cookie[];
}

// @public
export interface PaginatedList<Data> {
    count: number;
    desc?: boolean;
    items: Data[];
    limit: number;
    offset: number;
    total: number;
}

// @public (undocumented)
export interface ProcessedRequest {
    // (undocumented)
    requestId: string;
    // (undocumented)
    uniqueKey: string;
    // (undocumented)
    wasAlreadyHandled: boolean;
    // (undocumented)
    wasAlreadyPresent: boolean;
}

// @public
export interface ProxyInfo {
    hostname: string;
    ignoreTlsErrors?: boolean;
    password: string;
    port: number | string;
    url: string;
    username?: string;
}

// @public
export interface QueueOperationInfo {
    requestId: string;
    wasAlreadyHandled: boolean;
    wasAlreadyPresent: boolean;
}

// @public
export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
    url?: string | URL;
    headers: Headers;
}) => void;

// @public
export interface RequestQueueBackend {
    addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
    drop(): Promise<void>;
    fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
    getMetadata(): Promise<RequestQueueInfo>;
    getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
    isEmpty(): Promise<boolean>;
    isFinished(): Promise<boolean>;
    markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
    purge(): Promise<void>;
    reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
    setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}

// @public (undocumented)
export interface RequestQueueInfo {
    // (undocumented)
    accessedAt: Date;
    // (undocumented)
    createdAt: Date;
    // (undocumented)
    handledRequestCount: number;
    // (undocumented)
    id: string;
    // (undocumented)
    modifiedAt: Date;
    // (undocumented)
    name?: string;
    // (undocumented)
    pendingRequestCount: number;
    // (undocumented)
    totalRequestCount: number;
}

// @public
export interface RequestQueueOperationOptions {
    forefront?: boolean;
}

// @public (undocumented)
export interface RequestSchema {
    // (undocumented)
    errorMessages?: string[];
    // (undocumented)
    handledAt?: string;
    // (undocumented)
    headers?: Dictionary<string>;
    // (undocumented)
    id?: string;
    // (undocumented)
    loadedUrl?: string;
    // (undocumented)
    method?: AllowedHttpMethods;
    // (undocumented)
    noRetry?: boolean;
    // (undocumented)
    payload?: string;
    // (undocumented)
    retryCount?: number;
    // (undocumented)
    uniqueKey: string;
    // (undocumented)
    url: string;
    // (undocumented)
    userData?: Dictionary;
}

// @public (undocumented)
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;

// @public (undocumented)
export interface SendRequestOptions {
    // (undocumented)
    cookieJar?: CookieJar;
    ignoreTlsErrors?: boolean;
    proxyUrl?: string;
    // (undocumented)
    session?: ISession;
    signal?: AbortSignal;
    timeoutMillis?: number;
}

// @public
export interface SerializedCookieJar {
    // (undocumented)
    [key: string]: unknown;
    // (undocumented)
    cookies: Record<string, unknown>[];
    // (undocumented)
    rejectPublicSuffixes: boolean;
    // (undocumented)
    storeType: string | null;
    // (undocumented)
    version: string;
}

// @public
export interface SessionCookie {
    // (undocumented)
    canonicalizedDomain(): string | undefined;
    // (undocumented)
    cdomain(): string | undefined;
    // (undocumented)
    clone(): SessionCookie | undefined;
    // (undocumented)
    cookieString(): string;
    // (undocumented)
    creation: Date | 'Infinity' | null;
    // (undocumented)
    creationIndex: number;
    // (undocumented)
    domain: string | null;
    // (undocumented)
    expires: Date | 'Infinity' | null;
    // (undocumented)
    expiryDate(now?: Date): Date | undefined;
    // (undocumented)
    expiryTime(now?: Date): number | undefined;
    // (undocumented)
    extensions: string[] | null;
    // (undocumented)
    hostOnly: boolean | null;
    // (undocumented)
    httpOnly: boolean;
    // (undocumented)
    isPersistent(): boolean;
    // (undocumented)
    key: string;
    // (undocumented)
    lastAccessed: Date | 'Infinity' | null;
    // (undocumented)
    maxAge: number | 'Infinity' | '-Infinity' | null;
    // (undocumented)
    path: string | null;
    // (undocumented)
    pathIsDefault: boolean | null;
    // (undocumented)
    sameSite: string | undefined;
    // (undocumented)
    secure: boolean;
    // (undocumented)
    setExpires(exp: string | Date): void;
    // (undocumented)
    setMaxAge(age: number): void;
    // (undocumented)
    toJSON(): Record<string, unknown>;
    // (undocumented)
    toString(): string;
    // (undocumented)
    TTL(now?: number): number;
    // (undocumented)
    validate(): boolean;
    // (undocumented)
    value: string;
}

// @public
export interface SessionFingerprint {
    browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
    device?: 'desktop' | 'mobile';
    platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
}

// @public
export interface SessionState {
    // (undocumented)
    cookieJar: SerializedCookieJar;
    // (undocumented)
    createdAt: string;
    // (undocumented)
    errorScore: number;
    // (undocumented)
    errorScoreDecrement: number;
    // (undocumented)
    expiresAt: string;
    // (undocumented)
    fingerprint?: SessionFingerprint;
    // (undocumented)
    id: string;
    // (undocumented)
    maxErrorScore: number;
    // (undocumented)
    maxUsageCount: number;
    // (undocumented)
    proxyInfo?: ProxyInfo;
    // (undocumented)
    retired: boolean;
    // (undocumented)
    usageCount: number;
    // (undocumented)
    userData: object;
}

// @public
export interface SetStatusMessageOptions {
    isStatusMessageTerminal?: boolean;
    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
}

// @public
export interface StorageBackend {
    createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
    createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
    createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
    getStorageBackendCacheKey?(): string;
    purge?(): Promise<void>;
    // (undocumented)
    stats?: {
        rateLimitErrors: number[];
    };
    storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
    // (undocumented)
    teardown?(): Promise<void>;
}

// @public
export type StorageIdentifier = {
    id: string;
    name?: never;
    alias?: never;
} | {
    id?: never;
    name: string;
    alias?: never;
} | {
    id?: never;
    name?: never;
    alias: string;
} | {
    id?: never;
    name?: never;
    alias?: never;
};

// @public (undocumented)
export interface StreamOptions extends SendRequestOptions {
    // (undocumented)
    onRedirect?: RedirectHandler;
}

// @public (undocumented)
export interface UnprocessedRequest {
    // (undocumented)
    method?: AllowedHttpMethods;
    // (undocumented)
    uniqueKey: string;
    // (undocumented)
    url: string;
}

// @public (undocumented)
export interface UpdateRequestSchema extends RequestSchema {
    // (undocumented)
    id: string;
}

// (No @packageDocumentation comment for this package)