crawlee-core.api

Public API Report File for "@crawlee/core"

Do not edit this file. It is a report generated by API Extractor.

import type { AllowedHttpMethods } from '@crawlee/types';
import { ArgumentValidationError } from '@crawlee/utils';
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
import type { Awaitable } from '@crawlee/types';
import { BaseHttpClient } from '@crawlee/http-client';
import type { BatchAddRequestsResult } from '@crawlee/types';
import type { BetterIntervalID } from '@apify/utilities';
import type { Constructor } from '@crawlee/types';
import { CrawleeLogger } from '@crawlee/types';
import type { CrawleeLoggerOptions } from '@crawlee/types';
import type { DatasetBackend } from '@crawlee/types';
import type { DatasetInfo } from '@crawlee/types';
import { Dictionary } from '@crawlee/types';
import { EnqueueStrategy } from '@crawlee/utils';
import type { KeyValueStoreBackend } from '@crawlee/types';
import type { KeyValueStoreInfo } from '@crawlee/types';
import type { LiteralUnion } from 'type-fest';
import { Log } from '@apify/log';
import log from '@apify/log';
import { Logger } from '@apify/log';
import { LoggerJson } from '@apify/log';
import type { LoggerOptions } from '@apify/log';
import { LoggerText } from '@apify/log';
import { LogLevel } from '@apify/log';
import type { LogOptions } from '@crawlee/types';
import type { ProcessedRequest } from '@crawlee/types';
import type { ProxyInfo } from '@crawlee/types';
import type { QueueOperationInfo } from '@crawlee/types';
import type { ReadonlyDeep } from 'type-fest';
import type { RequestQueueBackend } from '@crawlee/types';
import type { RequestQueueInfo } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
import type * as storage from '@crawlee/types';
import { StorageBackend } from '@crawlee/types';
import { StorageIdentifier } from '@crawlee/types';
import { z } from 'zod';
import { ZodType } from 'zod';

// @public (undocumented)
export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions {
    batchSize?: number;
    maxNewRequests?: number;
    waitBetweenBatchesMillis?: number;
    waitForAllRequestsToBeAdded?: boolean;
}

// @public (undocumented)
export interface AddRequestsBatchedResult {
    // (undocumented)
    addedRequests: ProcessedRequest[];
    requestsOverLimit?: Source[];
    waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
}

// @public
export class AfterCommitError extends NonRetryableError {
    constructor(cause: unknown);
}

// @public
export class ApifyLogAdapter extends BaseCrawleeLogger {
    constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
    // (undocumented)
    protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    // (undocumented)
    logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
}

export { ArgumentValidationError }

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
class BaseClient {
    constructor(id: string);
    // (undocumented)
    id: string;
}

// @public
export abstract class BaseCrawleeLogger implements CrawleeLogger {
    constructor(options?: Partial<CrawleeLoggerOptions>);
    // (undocumented)
    child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
    // (undocumented)
    debug(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    deprecated(message: string): void;
    // (undocumented)
    error(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    exception(exception: Error, message: string, data?: Record<string, unknown>): void;
    // (undocumented)
    getOptions(): CrawleeLoggerOptions;
    // (undocumented)
    info(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
    // (undocumented)
    perf(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    setOptions(options: Partial<CrawleeLoggerOptions>): void;
    // (undocumented)
    softFail(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    warning(message: string, data?: Record<string, unknown>, options?: LogOptions): void;
    // (undocumented)
    warningOnce(message: string): void;
}

// @public
export const coerceBoolean: z.ZodPreprocess<z.ZodBoolean, unknown>;

// @public (undocumented)
export const coerceNumber: z.ZodPreprocess<z.ZodNumber, unknown>;

// @public (undocumented)
export interface ConfigField<T extends z.ZodType = z.ZodType> {
    // (undocumented)
    envVar?: string | string[];
    // (undocumented)
    schema: T;
}

// @public (undocumented)
export interface Configuration extends ResolvedConfigValues {
}

// @public
export class Configuration {
    constructor(options?: ConfigurationInput);
    protected static fields: Record<string, ConfigField>;
    static getGlobalConfiguration(): Configuration;
}

// @public (undocumented)
export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;

// @public @deprecated (undocumented)
export type ConfigurationOptions = ConfigurationInput;

// @public (undocumented)
export const crawleeConfigFields: {
    defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
    purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
    defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
    maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
    inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
    headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
    defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
    disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>, unknown>>>;
    persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
    storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
    containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
};

export { CrawleeLogger }

export { CrawleeLoggerOptions }

// @public
export function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;

// @public
export class CriticalError extends NonRetryableError {
}

// @public
export class Dataset<Data extends Dictionary = Dictionary> {
    [Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
    // (undocumented)
    backend: DatasetBackend<Data>;
    // (undocumented)
    readonly configuration: Configuration;
    drop(): Promise<void>;
    entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
    export(options?: DatasetExportOptions): Promise<Data[]>;
    exportTo(key: string, options?: DatasetExportToOptions, contentType?: string): Promise<Data[]>;
    exportToCSV(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
    static exportToCSV(key: string, options?: DatasetExportToOptions): Promise<void>;
    exportToJSON(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
    static exportToJSON(key: string, options?: DatasetExportToOptions): Promise<void>;
    forEach(iteratee: DatasetConsumer<Data>, options?: DatasetIteratorOptions, index?: number): Promise<void>;
    getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
    static getData<Data extends Dictionary = Dictionary>(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
    getInfo(): Promise<DatasetInfo>;
    // (undocumented)
    id: string;
    // (undocumented)
    log: CrawleeLogger;
    map<R>(iteratee: DatasetMapper<Data, R>, options?: DatasetIteratorOptions): Promise<R[]>;
    // (undocumented)
    name?: string;
    static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
    purge(): Promise<void>;
    pushData(data: Data | Data[]): Promise<void>;
    reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
    reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
    reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
    get stats(): DatasetStats;
    values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
}

// @public
export interface DatasetConsumer<Data> {
    // (undocumented)
    (item: Data, index: number): Awaitable<void>;
}

// @public (undocumented)
export interface DatasetContent<Data> {
    count: number;
    desc?: boolean;
    items: Data[];
    limit: number;
    offset: number;
    total: number;
}

// @public (undocumented)
export interface DatasetDataOptions {
    clean?: boolean;
    desc?: boolean;
    fields?: string[];
    limit?: number;
    offset?: number;
    skipEmpty?: boolean;
    skipHidden?: boolean;
    unwind?: string;
}

// @public (undocumented)
export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
    collectAllKeys?: boolean;
}

// @public (undocumented)
export interface DatasetExportToOptions extends DatasetExportOptions {
    // (undocumented)
    fromDataset?: string | StorageIdentifier;
    // (undocumented)
    toKVS?: string | StorageIdentifier;
}

// @public (undocumented)
export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
}

// @public
export interface DatasetJournalEntry {
    items: Dictionary[];
    // (undocumented)
    recordedAt: Date;
    // (undocumented)
    storageId: string;
    // (undocumented)
    type: 'dataset';
}

// @public
export interface DatasetMapper<Data, R> {
    (item: Data, index: number): Awaitable<R>;
}

// @public (undocumented)
export interface DatasetOptions {
    // (undocumented)
    backend: DatasetBackend;
    metadata: DatasetInfo;
}

// @public
export interface DatasetReducer<T, Data> {
    // (undocumented)
    (memo: T, item: Data, index: number): Awaitable<T>;
}

// @public
export interface DatasetStats {
    readCount: number;
    writeCount: number;
}

// @public
export interface DefaultStorageIdentifier {
    // (undocumented)
    alias?: never;
    // (undocumented)
    id?: never;
    // (undocumented)
    name?: never;
}

export { EnqueueStrategy }

// @public
export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';

// @public (undocumented)
export abstract class EventManager {
    constructor(options: EventManagerOptions);
    close(): Promise<void>;
    // (undocumented)
    emit(event: EventTypeName, ...args: unknown[]): void;
    // (undocumented)
    protected events: AsyncEventEmitter<{}>;
    init(): Promise<void>;
    // (undocumented)
    protected initialized: boolean;
    // (undocumented)
    protected intervals: Intervals;
    // (undocumented)
    isInitialized(): boolean;
    // (undocumented)
    protected log: CrawleeLogger;
    // (undocumented)
    off(event: EventTypeName, listener?: (...args: any[]) => any): void;
    // (undocumented)
    on(event: EventTypeName, listener: (...args: any[]) => any): void;
}

// @public (undocumented)
export interface EventManagerOptions {
    persistStateIntervalMillis: number;
}

// @public
export interface EventStatusMessageData {
    crawlerId: string;
    isStatusMessageTerminal?: boolean;
    level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
    message: string;
}

// @public (undocumented)
export enum EventType {
    // (undocumented)
    ABORTING = "aborting",
    // (undocumented)
    EXIT = "exit",
    // (undocumented)
    MIGRATING = "migrating",
    // (undocumented)
    PERSIST_STATE = "persistState",
    // (undocumented)
    STATUS_MESSAGE = "statusMessage",
    // (undocumented)
    SYSTEM_INFO = "systemInfo"
}

// @public (undocumented)
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';

// @public
export type ExplicitStorageIdentifier = {
    id: string;
    name?: never;
    alias?: never;
} | {
    id?: never;
    name: string;
    alias?: never;
} | {
    id?: never;
    name?: never;
    alias: string;
};

// @public (undocumented)
export function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;

// @public (undocumented)
export type FieldsInput<F extends Record<string, ConfigField>> = {
    [K in keyof F]?: z.output<F[K]['schema']>;
};

// @public (undocumented)
export type FieldsOutput<F extends Record<string, ConfigField>> = {
    [K in keyof F]: z.output<F[K]['schema']>;
};

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type Hashable = string;

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface Intervals {
    // (undocumented)
    persistState?: BetterIntervalID;
    // (undocumented)
    systemInfo?: BetterIntervalID;
}

// @public
export interface IProxyConfiguration {
    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
}

// @public
export interface IRequestLoader {
    [Symbol.asyncIterator](): AsyncGenerator<Request_2>;
    checkReadiness(): Promise<RequestSourceStatus>;
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
    getHandledCount(): Promise<number>;
    getPendingCount(): Promise<number>;
    getTotalCount(): Promise<number>;
    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
    persistState?(): Promise<void>;
    toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public
export interface IRequestManager extends IRequestLoader {
    // (undocumented)
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    // (undocumented)
    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
    purge?(): Promise<void>;
    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    recordPacingSignal(signal: PacingSignal): boolean;
    setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}

// @public
export interface IStorage {
    // (undocumented)
    id: string;
    // (undocumented)
    name?: string;
}

// @public
export interface JournaledRequest {
    // (undocumented)
    label?: string;
    snapshot?: Dictionary;
    // (undocumented)
    uniqueKey: string;
    // (undocumented)
    url: string;
}

// @public (undocumented)
export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;

// @public
export interface KeyConsumer {
    // (undocumented)
    (key: string, index: number, info: {
        size: number;
    }): Awaitable<void>;
}

// @public
export class KeyValueStore {
    [Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
    // (undocumented)
    readonly backend: KeyValueStoreBackend;
    // (undocumented)
    readonly configuration: Configuration;
    drop(): Promise<void>;
    entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
    forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
    // (undocumented)
    getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
    // (undocumented)
    static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
    getPublicUrl(key: string): Promise<string | undefined>;
    getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
    getValue<T = unknown>(key: string): Promise<T | null>;
    getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
    // (undocumented)
    readonly id: string;
    keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
    // (undocumented)
    readonly name?: string;
    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
    purge(): Promise<void>;
    recordExists(key: string): Promise<boolean>;
    static recordExists(key: string): Promise<boolean>;
    setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
    get stats(): KeyValueStoreStats;
    values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
}

// @public (undocumented)
export interface KeyValueStoreIteratorOptions {
    prefix?: string;
}

// @public
export interface KeyValueStoreJournalEntry {
    // (undocumented)
    key: string;
    // (undocumented)
    options?: RecordOptions;
    // (undocumented)
    storageId: string;
    // (undocumented)
    type: 'keyValueStore';
    value: unknown;
}

// @public (undocumented)
export interface KeyValueStoreOptions {
    // (undocumented)
    backend: KeyValueStoreBackend;
    metadata: KeyValueStoreInfo;
}

// @public
export interface KeyValueStoreRawRecord {
    // (undocumented)
    contentType: string | null;
    // (undocumented)
    value: Buffer | ArrayBuffer;
}

// @public
export interface KeyValueStoreStats {
    deleteCount: number;
    listCount: number;
    readCount: number;
    writeCount: number;
}

// @public (undocumented)
export interface LoadSignalInfo {
    // (undocumented)
    actualRatio: number;
    // (undocumented)
    isOverloaded: boolean;
    // (undocumented)
    limitRatio: number;
}

// @public (undocumented)
export class LocalEventManager extends EventManager {
    constructor(options: LocalEventManagerOptions);
    // (undocumented)
    close(): Promise<void>;
    static fromConfiguration(configuration?: Configuration): LocalEventManager;
    init(): Promise<void>;
}

// @public (undocumented)
export interface LocalEventManagerOptions extends EventManagerOptions {
    systemInfoIntervalMillis: number;
}

export { Log }

export { log }

export { Logger }

export { LoggerJson }

export { LoggerOptions }

export { LoggerText }

export { LogLevel }

export { LogOptions }

// @public (undocumented)
export class MemoryStorageBackend implements storage.StorageBackend {
    constructor(options?: MemoryStorageOptions);
    // (undocumented)
    createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
    // (undocumented)
    createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
    // (undocumented)
    createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend_2>;
    getStorageBackendCacheKey(): string;
    // (undocumented)
    readonly logger?: CrawleeLogger;
    purge(): Promise<void>;
    // (undocumented)
    storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
    teardown(): Promise<void>;
}

// @public (undocumented)
export interface MemoryStorageOptions {
    logger?: CrawleeLogger;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface NewUrlOptions {
    // (undocumented)
    request?: Request_2;
}

// @public
export class NonRetryableError extends Error {
}

// @public
export type PacingScope = LiteralUnion<'hostname' | 'registrableDomain', string>;

// @public
export type PacingSignal = {
    reason: 'rateLimited';
    url: string;
    waitMs?: number;
    scope?: PacingScope;
} | {
    reason: 'minInterval';
    url: string;
    intervalMs: number;
    scope: PacingScope;
} | {
    reason: 'minIntervalEverywhere';
    intervalMs: number;
    scope: PacingScope;
};

// @public
export function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;

// @public
export class ProxyConfiguration implements IProxyConfiguration {
    constructor(options?: ProxyConfigurationOptions);
    // (undocumented)
    readonly isManInTheMiddle = false;
    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
    newUrl(options?: NewUrlOptions): Promise<string | undefined>;
}

// @public (undocumented)
export interface ProxyConfigurationFunction {
    // (undocumented)
    (options?: {
        request?: Request_2;
    }): string | null | Promise<string | null>;
}

// @public (undocumented)
export interface ProxyConfigurationOptions {
    newUrlFunction?: ProxyConfigurationFunction;
    proxyUrls?: UrlList;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public
interface PurgeDefaultStorageOptions {
    // (undocumented)
    configuration?: Configuration;
    onlyPurgeOnce?: boolean;
    // (undocumented)
    storageBackend?: StorageBackend;
}

// @public
export function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;

// @public
export function purgeDefaultStorages(configuration?: Configuration, storageBackend?: StorageBackend): Promise<void>;

// @public (undocumented)
export interface PushErrorMessageOptions {
    omitStack?: boolean;
}

// @public (undocumented)
export interface RecordOptions {
    contentType?: string;
}

// @public
export class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
    constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
    get currentValue(): TStateModel;
    initialize(): Promise<TStateModel>;
    persistState(eventData?: Record<string, unknown>): Promise<void>;
    reset(): void;
    resetStore(): Promise<void>;
    teardown(): Promise<void>;
}

// @public
export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
    configuration?: Configuration;
    defaultState: TStateModel | (() => TStateModel);
    deserialize?: StateConversion<TPersistedState, TStateModel>;
    logger?: CrawleeLogger;
    serialize?: StateConversion<TStateModel, TPersistedState>;
}

// @public (undocumented)
export interface RecoverableStatePersistenceOptions {
    keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
    persistenceEnabled?: boolean;
    persistenceTimeoutMillis?: number;
    persistStateKey: string;
}

// @public
class Request_2<UserData extends Dictionary = Dictionary> {
    constructor(options: RequestOptions<UserData>);
    get crawlDepth(): number;
    set crawlDepth(value: number);
    errorMessages: string[];
    handledAt?: string;
    headers?: Record<string, string>;
    id?: string;
    intoFetchAPIRequest(): Request;
    get label(): string | undefined;
    set label(value: string | undefined);
    loadedUrl?: string;
    get maxRetries(): number | undefined;
    set maxRetries(value: number | undefined);
    method: AllowedHttpMethods;
    noRetry: boolean;
    payload?: string;
    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
    retryCount: number;
    get sessionId(): string | undefined;
    set sessionId(value: string | undefined);
    get skipNavigation(): boolean;
    set skipNavigation(value: boolean);
    get skippedReason(): SkippedRequestReason | undefined;
    set skippedReason(value: SkippedRequestReason | undefined);
    get state(): RequestState;
    set state(value: RequestState);
    uniqueKey: string;
    url: string;
    userData: UserData;
}
export { Request_2 as Request }

// @public
export class RequestList implements IRequestLoader {
    // (undocumented)
    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
    // (undocumented)
    checkReadiness(): Promise<RequestLoaderStatus>;
    // (undocumented)
    fetchNextRequest(): Promise<Request_2 | null>;
    // (undocumented)
    getHandledCount(): Promise<number>;
    getPendingCount(): Promise<number>;
    getState(): RequestListState;
    getTotalCount(): Promise<number>;
    // (undocumented)
    markRequestAsHandled(request: Request_2): Promise<void>;
    static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
    // (undocumented)
    persistState(): Promise<void>;
    teardown(): Promise<void>;
    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public (undocumented)
export interface RequestListOptions {
    httpClient?: BaseHttpClient;
    keepDuplicateUrls?: boolean;
    persistRequestsKey?: string;
    persistStateKey?: string;
    proxyConfiguration?: IProxyConfiguration;
    sources?: RequestListSource[];
    sourcesFunction?: RequestListSourcesFunction;
    state?: RequestListState;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type RequestListSource = string | Source;

// @public (undocumented)
export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;

// @public
export interface RequestListState {
    inProgress: string[];
    nextIndex: number;
    nextUniqueKey: string | null;
}

// @public
export type RequestLoaderStatus = Exclude<RequestSourceStatus, {
    status: 'stalled';
}>;

// @public
export class RequestManagerTandem implements IRequestManager {
    // (undocumented)
    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
    constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
    // (undocumented)
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    // (undocumented)
    addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
    checkReadiness(): Promise<RequestSourceStatus>;
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
    // (undocumented)
    getHandledCount(): Promise<number>;
    // (undocumented)
    getPendingCount(): Promise<number>;
    // (undocumented)
    getTotalCount(): Promise<number>;
    // (undocumented)
    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
    persistState(): Promise<void>;
    purge(): Promise<void>;
    // (undocumented)
    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    recordPacingSignal(signal: PacingSignal): boolean;
    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
}

// @public
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
    alwaysEnqueue?: boolean;
    crawlDepth?: number;
    headers?: Record<string, string>;
    keepUrlFragment?: boolean;
    label?: string;
    maxRetries?: number;
    method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
    noRetry?: boolean;
    payload?: string;
    sessionId?: string;
    skipNavigation?: boolean;
    uniqueKey?: string;
    url: string;
    useExtendedUniqueKey?: boolean;
    userData?: UserData;
}

// @public
export class RequestQueue implements IStorage, IRequestManager {
    // (undocumented)
    [Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
    addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
    addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
    addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
    // (undocumented)
    readonly backend: RequestQueueBackend;
    checkReadiness(): Promise<RequestLoaderStatus>;
    drop(): Promise<void>;
    fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
    getHandledCount(): Promise<number>;
    getInfo(): Promise<RequestQueueInfo>;
    getPendingCount(): Promise<number>;
    getRequest<T extends Dictionary = Dictionary>(uniqueKey: string): Promise<Request_2<T> | null>;
    getTotalCount(): Promise<number>;
    // (undocumented)
    readonly id: string;
    // (undocumented)
    readonly log: CrawleeLogger;
    markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | null>;
    // (undocumented)
    readonly name?: string;
    static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
    purge(): Promise<void>;
    reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
    recordPacingSignal(_signal: PacingSignal): boolean;
    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
    get stats(): RequestQueueStats;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
class RequestQueueBackend_2 extends BaseClient implements storage.RequestQueueBackend {
    constructor(options: RequestQueueBackendOptions);
    // (undocumented)
    accessedAt: Date;
    // (undocumented)
    addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
    cacheKey: string;
    // (undocumented)
    createdAt: Date;
    // (undocumented)
    drop(): Promise<void>;
    // (undocumented)
    fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
    // (undocumented)
    getMetadata(): Promise<storage.RequestQueueInfo>;
    // (undocumented)
    getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
    // (undocumented)
    handledRequestCount: number;
    // (undocumented)
    isEmpty(): Promise<boolean>;
    // (undocumented)
    isFinished(): Promise<boolean>;
    listItems(): Promise<storage.UpdateRequestSchema[]>;
    // (undocumented)
    markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
    // (undocumented)
    modifiedAt: Date;
    // (undocumented)
    name?: string;
    // (undocumented)
    pendingRequestCount: number;
    // (undocumented)
    purge(): Promise<void>;
    // (undocumented)
    reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
    // (undocumented)
    toRequestQueueInfo(): storage.RequestQueueInfo;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface RequestQueueBackendOptions {
    cacheKey?: string;
    // (undocumented)
    id?: string;
    // (undocumented)
    name?: string;
    // (undocumented)
    storageBackend: MemoryStorageBackend;
}

// @public
export interface RequestQueueJournalEntry {
    // (undocumented)
    forefront: boolean;
    // (undocumented)
    requests: JournaledRequest[];
    // (undocumented)
    type: 'requestQueue';
    writeThrough: boolean;
}

// @public (undocumented)
export interface RequestQueueOperationInfo extends QueueOperationInfo {
    // (undocumented)
    forefront: boolean;
    // (undocumented)
    uniqueKey: string;
}

// @public (undocumented)
export interface RequestQueueOperationOptions {
    forefront?: boolean;
}

// @public (undocumented)
export interface RequestQueueOptions {
    // (undocumented)
    backend: RequestQueueBackend;
    metadata: RequestQueueInfo;
    proxyConfiguration?: IProxyConfiguration;
}

// @public
export interface RequestQueueStats {
    headItemReadCount: number;
    writeCount: number;
}

// @public (undocumented)
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];

// @public
export type RequestSourceStatus = {
    status: 'ready';
} | {
    status: 'waiting';
    readyAt?: number;
} | {
    status: 'stalled';
    reason: string;
} | {
    status: 'finished';
};

// @public (undocumented)
export enum RequestState {
    // (undocumented)
    AFTER_NAV = 2,
    // (undocumented)
    BEFORE_NAV = 1,
    // (undocumented)
    DONE = 4,
    // (undocumented)
    ERROR = 6,
    // (undocumented)
    ERROR_HANDLER = 5,
    // (undocumented)
    REQUEST_HANDLER = 3,
    // (undocumented)
    SKIPPED = 7,
    // (undocumented)
    UNPROCESSED = 0
}

// @public
export class RequestValidationError extends NonRetryableError {
    constructor(label: string | symbol, issues: readonly SchemaIssue[]);
    // (undocumented)
    readonly issues: readonly SchemaIssue[];
    // (undocumented)
    readonly label: string | symbol;
}

// @public (undocumented)
export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;

// @public
export function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, storageBackend: StorageBackend, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;

// @public
export interface SchemaIssue {
    // (undocumented)
    readonly message: string;
    // (undocumented)
    readonly path?: readonly (PropertyKey | {
        key: PropertyKey;
    })[];
}

// @public
export function serializeValue(value: unknown, contentType?: string): {
    value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
    contentType: string;
};

// @public
export class ServiceConflictError extends Error {
    constructor(serviceName: string, newValue: unknown, existingValue: unknown);
}

// @public
export class ServiceLocator implements ServiceLocatorInterface {
    constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
    // (undocumented)
    getChildLog(prefix: string): CrawleeLogger;
    // (undocumented)
    getConfiguration(): Configuration;
    // (undocumented)
    getEventManager(): EventManager;
    // (undocumented)
    getLogger(): CrawleeLogger;
    // (undocumented)
    getStorageBackend(): StorageBackend;
    // (undocumented)
    getStorageInstanceManager(): StorageInstanceManager;
    // (undocumented)
    reset(): void;
    // (undocumented)
    setConfiguration(configuration: Configuration): void;
    // (undocumented)
    setEventManager(eventManager: EventManager): void;
    // (undocumented)
    setLogger(logger: CrawleeLogger): void;
    // (undocumented)
    setStorageBackend(storageBackend: StorageBackend): void;
}

// @public (undocumented)
export const serviceLocator: ServiceLocatorInterface;

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface ServiceLocatorInterface {
    getChildLog(prefix: string): CrawleeLogger;
    getConfiguration(): Configuration;
    getEventManager(): EventManager;
    getLogger(): CrawleeLogger;
    getStorageBackend(): StorageBackend;
    getStorageInstanceManager(): StorageInstanceManager;
    setConfiguration(configuration: Configuration): void;
    setEventManager(eventManager: EventManager): void;
    setLogger(logger: CrawleeLogger): void;
    setStorageBackend(storageBackend: StorageBackend): void;
}

// @public
export class SessionError extends Error {
    constructor(message?: string);
}

// @public (undocumented)
export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';

// @public (undocumented)
export type Source = (Partial<RequestOptions> & {
    requestsFromUrl?: string;
    regex?: RegExp;
}) | Request_2;

// @public
export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;

// @public
export class StateValidationError extends Error {
    constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
    // (undocumented)
    readonly issues: readonly SchemaIssue[];
    // (undocumented)
    readonly persistStateKey: string;
}

export { StorageBackend }

export { StorageIdentifier }

// @public
export class StorageInstanceManager {
    clearCache(): void;
    openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, input: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
        backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
        backendCacheKey: Hashable;
    }): Promise<TStorage>;
    removeFromCache(instance: IStorage): void;
}

// @public
export interface StorageOpenOptions {
    configuration?: Configuration;
    httpClient?: BaseHttpClient;
    proxyConfiguration?: IProxyConfiguration;
    storageBackend?: StorageBackend;
}

// @public
export class StorageStatsTracker<T extends Record<keyof T, number>> {
    constructor(initial: T);
    add(key: keyof T, by?: number): void;
    get current(): T;
}

// @public
export class StorageTransaction implements StorageTransactionView {
    afterCommit(callback: (error?: Error) => Awaitable<void>): void;
    commit(): Promise<void>;
    // (undocumented)
    get datasetItems(): {
        item: Dictionary;
        datasetId: string;
    }[];
    dispose(): void;
    // (undocumented)
    get enqueuedUrls(): {
        url: string;
        label?: string;
    }[];
    get isActive(): boolean;
    readonly journal: JournalEntry[];
    // (undocumented)
    get keyValueStoreChanges(): Record<string, Record<string, {
        changedValue: unknown;
        options?: RecordOptions;
    }>>;
    readonly policy: StorageWritePolicy;
    rollback(): void;
    run<T>(callback: () => Awaitable<T>): Promise<T>;
    // (undocumented)
    get state(): StorageTransactionState;
}

// @public (undocumented)
export interface StorageTransactionOptions {
    commitTimeoutMillis?: number;
    policy?: Partial<StorageWritePolicy>;
}

// @public (undocumented)
export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';

// @public
export interface StorageTransactionView {
    readonly datasetItems: {
        item: Dictionary;
        datasetId: string;
    }[];
    readonly enqueuedUrls: {
        url: string;
        label?: string;
    }[];
    readonly keyValueStoreChanges: Record<string, Record<string, {
        changedValue: unknown;
        options?: RecordOptions;
    }>>;
    // (undocumented)
    readonly state: StorageTransactionState;
}

// @public
export type StorageWriteMode = 'deferred' | 'writeThrough';

// @public
export interface StorageWritePolicy {
    requestQueue: StorageWriteMode;
}

// @public
export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;

// @public
export interface SystemInfo {
    // (undocumented)
    cpuInfo: LoadSignalInfo;
    // (undocumented)
    eventLoopInfo: LoadSignalInfo;
    isSystemIdle: boolean;
    loadSignalInfo?: Record<string, LoadSignalInfo>;
    // (undocumented)
    memCurrentBytes?: number;
    // (undocumented)
    memInfo: LoadSignalInfo;
    // (undocumented)
    memTotalBytes?: number;
    // (undocumented)
    storageBackendInfo: LoadSignalInfo;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type UrlList = (string | null)[];

// @public
export function useState<State extends Dictionary = Dictionary>(name?: string, defaultValue?: State, options?: UseStateOptions): Promise<State>;

// @public (undocumented)
export interface UseStateOptions {
    // (undocumented)
    configuration?: Configuration;
    keyValueStoreName?: string | null;
}

// @public
export function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;

// @public
export function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;

// (No @packageDocumentation comment for this package)