Skip to main content
Version: 4.0 (RC)

crawlee-core.api

Public API Report File for "@crawlee/core"

Do not edit this file. It is a report generated by API Extractor.


import type { AllowedHttpMethods } from '@crawlee/types';
import { ArgumentValidationError } from '@crawlee/utils';
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
import type { Awaitable } from '@crawlee/types';
import { BaseHttpClient } from '@crawlee/http-client';
import type { BatchAddRequestsResult } from '@crawlee/types';
import type { BetterIntervalID } from '@apify/utilities';
import type { Constructor } from '@crawlee/types';
import { CookieJar } from 'tough-cookie';
import { CrawleeLogger } from '@crawlee/types';
import type { CrawleeLoggerOptions } from '@crawlee/types';
import type { DatasetBackend } from '@crawlee/types';
import type { DatasetInfo } from '@crawlee/types';
import { Dictionary } from '@crawlee/types';
import { EnqueueStrategy } from '@crawlee/utils';
import type { HttpRequestOptions } from '@crawlee/types';
import type { ISession } from '@crawlee/types';
import type { ISessionPool } from '@crawlee/types';
import type { KeyValueStoreBackend } from '@crawlee/types';
import type { KeyValueStoreInfo } from '@crawlee/types';
import { Log } from '@apify/log';
import log from '@apify/log';
import { Logger } from '@apify/log';
import { LoggerJson } from '@apify/log';
import type { LoggerOptions } from '@apify/log';
import { LoggerText } from '@apify/log';
import { LogLevel } from '@apify/log';
import { ParseSitemapOptions } from '@crawlee/utils';
import type { ProcessedRequest } from '@crawlee/types';
import type { ProxyInfo } from '@crawlee/types';
import type { QueueOperationInfo } from '@crawlee/types';
import type { ReadonlyDeep } from 'type-fest';
import type { RequestQueueBackend } from '@crawlee/types';
import type { RequestQueueInfo } from '@crawlee/types';
import type { SendRequestOptions } from '@crawlee/types';
import type { SessionFingerprint } from '@crawlee/types';
import { SessionState } from '@crawlee/types';
import type { StandardSchemaV1 } from '@standard-schema/spec';
import type * as storage from '@crawlee/types';
import { StorageBackend } from '@crawlee/types';
import { StorageIdentifier } from '@crawlee/types';
import { z } from 'zod';
import { ZodType } from 'zod';

// @public (undocumented)
export interface AddRequestsBatchedOptions extends RequestQueueOperationOptions {
batchSize?: number;
maxNewRequests?: number;
waitBetweenBatchesMillis?: number;
waitForAllRequestsToBeAdded?: boolean;
}

// @public (undocumented)
export interface AddRequestsBatchedResult {
// (undocumented)
addedRequests: ProcessedRequest[];
requestsOverLimit?: Source[];
waitForAllRequestsToBeAdded: Promise<ProcessedRequest[]>;
}

// @public
export class ApifyLogAdapter extends BaseCrawleeLogger {
constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
// (undocumented)
protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
// (undocumented)
logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
}

export { ArgumentValidationError }

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
class BaseClient {
constructor(id: string);
// (undocumented)
id: string;
}

// @public
export abstract class BaseCrawleeLogger implements CrawleeLogger {
constructor(options?: Partial<CrawleeLoggerOptions>);
// (undocumented)
child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
protected abstract createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
// (undocumented)
debug(message: string, data?: Record<string, unknown>): void;
// (undocumented)
deprecated(message: string): void;
// (undocumented)
error(message: string, data?: Record<string, unknown>): void;
// (undocumented)
exception(exception: Error, message: string, data?: Record<string, unknown>): void;
// (undocumented)
getOptions(): CrawleeLoggerOptions;
// (undocumented)
info(message: string, data?: Record<string, unknown>): void;
abstract logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
// (undocumented)
perf(message: string, data?: Record<string, unknown>): void;
// (undocumented)
setOptions(options: Partial<CrawleeLoggerOptions>): void;
// (undocumented)
softFail(message: string, data?: Record<string, unknown>): void;
// (undocumented)
warning(message: string, data?: Record<string, unknown>): void;
// (undocumented)
warningOnce(message: string): void;
}

// @public (undocumented)
export const BLOCKED_STATUS_CODES: number[];

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface BrowserCrawlingContext {
// (undocumented)
saveSnapshot: (options: {
key: string;
}) => Promise<void>;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface BrowserPage {
// (undocumented)
content: () => Promise<string>;
}

// @public
export interface CalculatedStatistics {
crawlerRuntimeMillis: number;
requestAvgFailedDurationMillis: number;
requestAvgFinishedDurationMillis: number;
requestsFailedPerMinute: number;
requestsFinishedPerMinute: number;
requestsTotal: number;
requestTotalDurationMillis: number;
}

// @public
export const coerceBoolean: z.ZodPreprocess<z.ZodBoolean>;

// @public (undocumented)
export const coerceNumber: z.ZodPreprocess<z.ZodNumber>;

// @public
export interface ConcurrencyConsumer {
readonly id: string;
}

// @public
export class ConcurrencySystem implements IConcurrencySystem {
// (undocumented)
[Symbol.asyncDispose](): Promise<void>;
constructor(options?: ConcurrencySystemOptions);
// (undocumented)
get currentConcurrency(): number;
get desiredConcurrency(): number;
set desiredConcurrency(value: number);
getCurrentStatus(): SystemInfo;
hasCapacityForTask(_consumer?: ConcurrencyConsumer): boolean;
get isRunning(): boolean;
get maxConcurrency(): number;
set maxConcurrency(value: number);
get minConcurrency(): number;
set minConcurrency(value: number);
registerTaskEnd(_consumer?: ConcurrencyConsumer): void;
start(): Promise<void>;
stop(): Promise<void>;
tryRegisterTaskStart(consumer?: ConcurrencyConsumer): boolean;
}

// @public (undocumented)
export interface ConcurrencySystemOptions {
autoscaleIntervalSecs?: number;
currentHistorySecs?: number;
desiredConcurrency?: number;
desiredConcurrencyRatio?: number;
loadSignals?: LoadSignalsOptions;
// (undocumented)
log?: CrawleeLogger;
loggingIntervalSecs?: number | null;
maxConcurrency?: number;
maxTasksPerMinute?: number;
minConcurrency?: number;
scaleDownStepRatio?: number;
scaleUpStepRatio?: number;
snapshotHistorySecs?: number;
}

// @public (undocumented)
export interface ConfigField<T extends z.ZodType = z.ZodType> {
// (undocumented)
envVar?: string | string[];
// (undocumented)
schema: T;
}

// @public (undocumented)
export interface Configuration extends ResolvedConfigValues {
}

// @public
export class Configuration {
constructor(options?: ConfigurationInput);
protected static fields: Record<string, ConfigField>;
static getGlobalConfiguration(): Configuration;
}

// @public (undocumented)
export type ConfigurationInput = FieldsInput<typeof crawleeConfigFields>;

// @public @deprecated (undocumented)
export type ConfigurationOptions = ConfigurationInput;

// @public
export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
}

// @public
export abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
}

// @public (undocumented)
export class ContextPipelineCleanupError extends CriticalError {
constructor(error: unknown, options?: ErrorOptions);
}

// @public (undocumented)
export class ContextPipelineInitializationError extends Error {
constructor(error: unknown, options?: ErrorOptions);
}

// @public (undocumented)
export class ContextPipelineInterruptedError extends Error {
constructor(message?: string);
}

// @public
export class CpuLoadSignal implements LoadSignal {
constructor(options?: CpuLoadSignalOptions);
// (undocumented)
getSample(sampleDurationMillis?: number): LoadSnapshot[];
// (undocumented)
readonly name = "cpuInfo";
// (undocumented)
readonly overloadedRatio: number;
// (undocumented)
start(context: LoadSignalStartContext): Promise<void>;
// (undocumented)
stop(): Promise<void>;
}

// @public
export interface CpuLoadSignalOptions {
overloadedRatio?: number;
}

// @public (undocumented)
export const crawleeConfigFields: {
defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof LogLevel>>>>;
persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
};

export { CrawleeLogger }

export { CrawleeLoggerOptions }

// @public (undocumented)
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
extendTimeout(secs: number): void;
registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
}

// @public
export interface CreateSession {
// (undocumented)
(options?: {
sessionOptions?: SessionOptions;
}): Session | Promise<Session>;
}

// @public
export function createStorageTransaction(options?: StorageTransactionOptions): StorageTransaction;

// @public
export class CriticalError extends NonRetryableError {
}

// @public
export class Dataset<Data extends Dictionary = Dictionary> {
[Symbol.asyncIterator](): AsyncGenerator<Data, void, undefined>;
// (undocumented)
backend: DatasetBackend<Data>;
// (undocumented)
readonly configuration: Configuration;
drop(): Promise<void>;
entries(options?: DatasetIteratorOptions): AsyncIterable<[number, Data]> & Promise<[number, Data][]>;
export(options?: DatasetExportOptions): Promise<Data[]>;
exportTo(key: string, options?: DatasetExportToOptions, contentType?: string): Promise<Data[]>;
exportToCSV(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
static exportToCSV(key: string, options?: DatasetExportToOptions): Promise<void>;
exportToJSON(key: string, options?: Omit<DatasetExportToOptions, 'fromDataset'>): Promise<void>;
static exportToJSON(key: string, options?: DatasetExportToOptions): Promise<void>;
forEach(iteratee: DatasetConsumer<Data>, options?: DatasetIteratorOptions, index?: number): Promise<void>;
getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
static getData<Data extends Dictionary = Dictionary>(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
getInfo(): Promise<DatasetInfo>;
// (undocumented)
id: string;
// (undocumented)
log: CrawleeLogger;
map<R>(iteratee: DatasetMapper<Data, R>, options?: DatasetIteratorOptions): Promise<R[]>;
// (undocumented)
name?: string;
static open<Data extends Dictionary = Dictionary>(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<Dataset<Data>>;
pushData(data: Data | Data[]): Promise<void>;
reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
get stats(): DatasetStats;
values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
}

// @public
export interface DatasetConsumer<Data> {
// (undocumented)
(item: Data, index: number): Awaitable<void>;
}

// @public (undocumented)
export interface DatasetContent<Data> {
count: number;
desc?: boolean;
items: Data[];
limit: number;
offset: number;
total: number;
}

// @public (undocumented)
export interface DatasetDataOptions {
clean?: boolean;
desc?: boolean;
fields?: string[];
limit?: number;
offset?: number;
skipEmpty?: boolean;
skipHidden?: boolean;
unwind?: string;
}

// @public (undocumented)
export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
collectAllKeys?: boolean;
}

// @public (undocumented)
export interface DatasetExportToOptions extends DatasetExportOptions {
// (undocumented)
fromDataset?: string | StorageIdentifier;
// (undocumented)
toKVS?: string | StorageIdentifier;
}

// @public (undocumented)
export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
}

// @public
export interface DatasetJournalEntry {
items: Dictionary[];
// (undocumented)
recordedAt: Date;
// (undocumented)
storageId: string;
// (undocumented)
type: 'dataset';
}

// @public
export interface DatasetMapper<Data, R> {
(item: Data, index: number): Awaitable<R>;
}

// @public (undocumented)
export interface DatasetOptions {
// (undocumented)
backend: DatasetBackend;
metadata: DatasetInfo;
}

// @public
export interface DatasetReducer<T, Data> {
// (undocumented)
(memo: T, item: Data, index: number): Awaitable<T>;
}

// @public
export interface DatasetStats {
readCount: number;
writeCount: number;
}

// @public
export const defaultRoute: unique symbol;

// @public
export type DefaultRouteUserData<Routes, Fallback extends Dictionary> = Routes extends {
[defaultRoute]: infer DefaultUserData extends Dictionary;
} ? DefaultUserData : Fallback;

// @public
export interface DefaultStorageIdentifier {
// (undocumented)
alias?: never;
// (undocumented)
id?: never;
// (undocumented)
name?: never;
}

// @public
export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;

export { EnqueueStrategy }

// @public
export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';

// @public
export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
baseUrl?: string;
exclude?: readonly UrlPatternInput[];
include?: readonly UrlPatternInput[];
label?: string;
limit?: number;
onSkippedRequest?: SkippedRequestCallback;
sessionId?: string;
skipNavigation?: boolean;
strategy?: EnqueueStrategyOption;
transformRequestFunction?: RequestTransform;
userData?: Dictionary;
waitForAllRequestsToBeAdded?: boolean;
}

// @public
export interface ErrnoException extends Error {
// (undocumented)
cause?: any;
// (undocumented)
code?: string | number;
// (undocumented)
errno?: number;
// (undocumented)
path?: string;
// (undocumented)
syscall?: string;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface ErrorSnapshot {
// (undocumented)
htmlFileName?: string;
// (undocumented)
htmlFileUrl?: string;
// (undocumented)
screenshotFileName?: string;
// (undocumented)
screenshotFileUrl?: string;
}

// @public
export class ErrorSnapshotter {
// (undocumented)
static readonly BASE_MESSAGE = "An error occurred";
captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
generateFilename(error: ErrnoException): string;
// (undocumented)
static readonly MAX_ERROR_CHARACTERS = 30;
// (undocumented)
static readonly MAX_FILENAME_LENGTH = 250;
// (undocumented)
static readonly MAX_HASH_LENGTH = 30;
saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
// (undocumented)
static readonly SNAPSHOT_PREFIX = "ERROR_SNAPSHOT";
}

// @public
export class ErrorTracker {
constructor(options?: Partial<ErrorTrackerOptions>);
// (undocumented)
add(error: ErrnoException): void;
addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
// (undocumented)
captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
// (undocumented)
errorSnapshotter?: ErrorSnapshotter;
// (undocumented)
getMostPopularErrors(count: number): [number, string[]][];
// (undocumented)
getUniqueErrorCount(): number;
// (undocumented)
reset(): void;
// (undocumented)
result: Record<string, unknown>;
// (undocumented)
total: number;
}

// @public (undocumented)
export interface ErrorTrackerOptions {
// (undocumented)
saveErrorSnapshots: boolean;
// (undocumented)
showErrorCode: boolean;
// (undocumented)
showErrorMessage: boolean;
// (undocumented)
showErrorName: boolean;
// (undocumented)
showFullMessage: boolean;
// (undocumented)
showFullStack: boolean;
// (undocumented)
showStackTrace: boolean;
}

// @public
export class EventLoopLoadSignal implements LoadSignal {
constructor(options?: EventLoopLoadSignalOptions);
// (undocumented)
getSample(sampleDurationMillis?: number): LoadSnapshot[];
// (undocumented)
readonly name = "eventLoopInfo";
// (undocumented)
readonly overloadedRatio: number;
// (undocumented)
start(context: LoadSignalStartContext): Promise<void>;
// (undocumented)
stop(): Promise<void>;
}

// @public
export interface EventLoopLoadSignalOptions {
maxBlockedMillis?: number;
overloadedRatio?: number;
snapshotIntervalSecs?: number;
}

// @public (undocumented)
export abstract class EventManager {
constructor(options: EventManagerOptions);
close(): Promise<void>;
// (undocumented)
emit(event: EventTypeName, ...args: unknown[]): void;
// (undocumented)
protected events: AsyncEventEmitter<{}>;
init(): Promise<void>;
// (undocumented)
protected initialized: boolean;
// (undocumented)
protected intervals: Intervals;
// (undocumented)
isInitialized(): boolean;
// (undocumented)
protected log: CrawleeLogger;
// (undocumented)
off(event: EventTypeName, listener?: (...args: any[]) => any): void;
// (undocumented)
on(event: EventTypeName, listener: (...args: any[]) => any): void;
}

// @public (undocumented)
export interface EventManagerOptions {
persistStateIntervalMillis: number;
}

// @public
export interface EventStatusMessageData {
crawlerId: string;
isStatusMessageTerminal?: boolean;
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
message: string;
}

// @public (undocumented)
export enum EventType {
// (undocumented)
ABORTING = "aborting",
// (undocumented)
EXIT = "exit",
// (undocumented)
MIGRATING = "migrating",
// (undocumented)
PERSIST_STATE = "persistState",
// (undocumented)
STATUS_MESSAGE = "statusMessage",
// (undocumented)
SYSTEM_INFO = "systemInfo"
}

// @public (undocumented)
export type EventTypeName = EventType | 'systemInfo' | 'persistState' | 'migrating' | 'aborting' | 'exit' | 'statusMessage';

// @public
export type ExplicitStorageIdentifier = {
id: string;
name?: never;
alias?: never;
} | {
id?: never;
name: string;
alias?: never;
} | {
id?: never;
name?: never;
alias: string;
};

// @public
export interface ExtractLinksOptions {
baseUrl?: string;
selector?: string;
}

// @public (undocumented)
export function field<T extends z.ZodType>(schema: T, envVar?: string | string[]): ConfigField<T>;

// @public (undocumented)
export type FieldsInput<F extends Record<string, ConfigField>> = {
[K in keyof F]?: z.output<F[K]['schema']>;
};

// @public (undocumented)
export type FieldsOutput<F extends Record<string, ConfigField>> = {
[K in keyof F]: z.output<F[K]['schema']>;
};

// @public (undocumented)
export interface FinalStatistics {
// (undocumented)
crawlerRuntimeMillis: number;
// (undocumented)
requestAvgFailedDurationMillis: number;
// (undocumented)
requestAvgFinishedDurationMillis: number;
// (undocumented)
requestsFailed: number;
// (undocumented)
requestsFailedPerMinute: number;
// (undocumented)
requestsFinished: number;
// (undocumented)
requestsFinishedPerMinute: number;
// (undocumented)
requestsTotal: number;
// (undocumented)
requestTotalDurationMillis: number;
// (undocumented)
retryHistogram: number[];
}

// @public (undocumented)
export type GetUserDataFromRequest<T> = T extends Request_2<infer Y> ? Y : never;

// @public (undocumented)
export type GlobInput = string | GlobObject;

// @public (undocumented)
export interface GlobObject {
// (undocumented)
glob: string;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type Hashable = string;

// @public
export interface IConcurrencySystem {
readonly currentConcurrency: number;
readonly desiredConcurrency: number;
hasCapacityForTask(consumer: ConcurrencyConsumer): boolean;
readonly isRunning: boolean;
registerTaskEnd(consumer: ConcurrencyConsumer): void;
tryRegisterTaskStart(consumer: ConcurrencyConsumer): boolean;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface Intervals {
// (undocumented)
persistState?: BetterIntervalID;
// (undocumented)
systemInfo?: BetterIntervalID;
}

// @public
export interface IProxyConfiguration {
newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
}

// @public
export interface IRequestLoader {
[Symbol.asyncIterator](): AsyncGenerator<Request_2>;
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
getHandledCount(): Promise<number>;
getPendingCount(): Promise<number>;
getTotalCount(): Promise<number>;
isEmpty(): Promise<boolean>;
isFinished(): Promise<boolean>;
markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
persistState?(): Promise<void>;
toTandem?(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public
export interface IRequestManager extends IRequestLoader {
// (undocumented)
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
// (undocumented)
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
purge?(): Promise<void>;
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
}

// @public
export interface IStatistics<StateExtension extends object = {}> {
calculate(): CalculatedStatistics;
discardJob(id: number | string): void;
readonly errorTracker: ErrorTracker;
readonly errorTrackerRetry: ErrorTracker;
failJob(id: number | string, retryCount: number): void;
finishJob(id: number | string, retryCount: number): void;
persistState?(): Promise<void>;
registerStatusCode(code: number): void;
readonly requestRetryHistogram: number[];
startCapturing(): Promise<void>;
startJob(id: number | string): void;
readonly state: StatisticState & StateExtension;
stopCapturing(): Promise<void>;
}

// @public
export interface IStorage {
// (undocumented)
id: string;
// (undocumented)
name?: string;
}

// @public
export interface JournaledRequest {
// (undocumented)
label?: string;
snapshot?: Dictionary;
// (undocumented)
uniqueKey: string;
// (undocumented)
url: string;
}

// @public (undocumented)
export type JournalEntry = DatasetJournalEntry | KeyValueStoreJournalEntry | RequestQueueJournalEntry;

// @public
export interface KeyConsumer {
// (undocumented)
(key: string, index: number, info: {
size: number;
}): Awaitable<void>;
}

// @public
export class KeyValueStore {
[Symbol.asyncIterator]<T = unknown>(): AsyncGenerator<[string, T], void, undefined>;
// (undocumented)
readonly configuration: Configuration;
drop(): Promise<void>;
entries<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<[string, T]> & Promise<[string, T][]>;
forEachKey(iteratee: KeyConsumer, options?: KeyValueStoreIteratorOptions): Promise<void>;
// (undocumented)
getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
// (undocumented)
static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
getPublicUrl(key: string): Promise<string | undefined>;
getRecord(key: string): Promise<KeyValueStoreRawRecord | null>;
getValue<T = unknown>(key: string): Promise<T | null>;
getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
// (undocumented)
readonly id: string;
keys(options?: KeyValueStoreIteratorOptions): AsyncIterable<string> & Promise<string[]>;
// (undocumented)
readonly name?: string;
static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<KeyValueStore>;
recordExists(key: string): Promise<boolean>;
static recordExists(key: string): Promise<boolean>;
setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
get stats(): KeyValueStoreStats;
values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
}

// @public (undocumented)
export interface KeyValueStoreIteratorOptions {
prefix?: string;
}

// @public
export interface KeyValueStoreJournalEntry {
// (undocumented)
key: string;
// (undocumented)
options?: RecordOptions;
// (undocumented)
storageId: string;
// (undocumented)
type: 'keyValueStore';
value: unknown;
}

// @public (undocumented)
export interface KeyValueStoreOptions {
// (undocumented)
backend: KeyValueStoreBackend;
metadata: KeyValueStoreInfo;
}

// @public
export interface KeyValueStoreRawRecord {
// (undocumented)
contentType: string | null;
// (undocumented)
value: Buffer | ArrayBuffer;
}

// @public
export interface KeyValueStoreStats {
deleteCount: number;
listCount: number;
readCount: number;
writeCount: number;
}

// @public
export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request_2 | ({
requestsFromUrl?: string;
regex?: RegExp;
} & ({
[Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
label: Label;
};
}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
label?: undefined;
})));

// @public (undocumented)
export type LoadedRequest<R extends Request_2> = WithRequired<R, 'id' | 'loadedUrl'>;

// @public
export interface LoadSignal {
getSample(sampleDurationMillis?: number): LoadSnapshot[];
readonly name: string;
readonly overloadedRatio: number;
start(context: LoadSignalStartContext): Promise<void>;
stop(): Promise<void>;
}

// @public (undocumented)
export interface LoadSignalInfo {
// (undocumented)
actualRatio: number;
// (undocumented)
isOverloaded: boolean;
// (undocumented)
limitRatio: number;
}

// @public
export interface LoadSignalsOptions {
cpu?: CpuLoadSignalOptions | false;
custom?: LoadSignal[];
eventLoop?: EventLoopLoadSignalOptions | false;
memory?: MemoryLoadSignalOptions | false;
storageBackend?: StorageBackendLoadSignalOptions | false;
}

// @public
export interface LoadSignalStartContext {
maxSampleWindowMillis: number;
}

// @public
export interface LoadSnapshot {
// (undocumented)
createdAt: Date;
// (undocumented)
isOverloaded: boolean;
}

// @public (undocumented)
export class LocalEventManager extends EventManager {
constructor(options: LocalEventManagerOptions);
// (undocumented)
close(): Promise<void>;
static fromConfiguration(configuration?: Configuration): LocalEventManager;
init(): Promise<void>;
}

// @public (undocumented)
export interface LocalEventManagerOptions extends EventManagerOptions {
systemInfoIntervalMillis: number;
}

export { Log }

export { log }

export { Logger }

export { LoggerJson }

export { LoggerOptions }

export { LoggerText }

export { LogLevel }

// @public (undocumented)
export const MAX_POOL_SIZE = 1000;

// @public
export class MemoryLoadSignal implements LoadSignal {
constructor(options?: MemoryLoadSignalOptions);
// (undocumented)
getSample(sampleDurationMillis?: number): LoadSnapshot[];
// (undocumented)
readonly name = "memInfo";
// (undocumented)
readonly overloadedRatio: number;
// (undocumented)
start(context: LoadSignalStartContext): Promise<void>;
// (undocumented)
stop(): Promise<void>;
}

// @public
export interface MemoryLoadSignalOptions {
maxUsedRatio?: number;
overloadedRatio?: number;
}

// @public (undocumented)
export class MemoryStorageBackend implements storage.StorageBackend {
constructor(options?: MemoryStorageOptions);
// (undocumented)
createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
// (undocumented)
createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
// (undocumented)
createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend_2>;
getStorageBackendCacheKey(): string;
// (undocumented)
readonly logger?: CrawleeLogger;
purge(): Promise<void>;
// (undocumented)
storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
teardown(): Promise<void>;
}

// @public (undocumented)
export interface MemoryStorageOptions {
logger?: CrawleeLogger;
}

// @public
export class MissingSessionError extends Error {
constructor(sessionId?: string);
}

// @public
export class NavigationSkippedError extends NonRetryableError {
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface NewUrlOptions {
// (undocumented)
request?: Request_2;
}

// @public
export class NonRetryableError extends Error {
}

// @public
export function parseRetryAfterHeader(value?: string | null): number | null;

// @public
export function parseValue(body: Buffer | ArrayBuffer | string, contentTypeHeader: string | null): string | Buffer | ArrayBuffer | Record<string, unknown>;

// @public (undocumented)
export const PERSIST_STATE_KEY = "CRAWLEE_SESSION_POOL_STATE";

// @public
export interface PersistenceOptions {
enable?: boolean;
}

// @public
export class PersistentRateLimitError extends CriticalError {
}

// @public
export class ProxyConfiguration implements IProxyConfiguration {
constructor(options?: ProxyConfigurationOptions);
// (undocumented)
readonly isManInTheMiddle = false;
newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
newUrl(options?: NewUrlOptions): Promise<string | undefined>;
}

// @public (undocumented)
export interface ProxyConfigurationFunction {
// (undocumented)
(options?: {
request?: Request_2;
}): string | null | Promise<string | null>;
}

// @public (undocumented)
export interface ProxyConfigurationOptions {
newUrlFunction?: ProxyConfigurationFunction;
proxyUrls?: UrlList;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public
interface PurgeDefaultStorageOptions {
// (undocumented)
configuration?: Configuration;
onlyPurgeOnce?: boolean;
// (undocumented)
storageBackend?: StorageBackend;
}

// @public
export function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;

// @public
export function purgeDefaultStorages(configuration?: Configuration, storageBackend?: StorageBackend): Promise<void>;

// @public (undocumented)
export interface PushErrorMessageOptions {
omitStack?: boolean;
}

// @public (undocumented)
export interface RecordOptions {
contentType?: string;
}

// @public
export class RecoverableState<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> {
constructor(options: RecoverableStateOptions<TStateModel, TPersistedState>);
get currentValue(): TStateModel;
initialize(): Promise<TStateModel>;
persistState(eventData?: Record<string, unknown>): Promise<void>;
reset(): void;
resetStore(): Promise<void>;
teardown(): Promise<void>;
}

// @public
export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
configuration?: Configuration;
defaultState: TStateModel | (() => TStateModel);
deserialize?: StateConversion<TPersistedState, TStateModel>;
logger?: CrawleeLogger;
serialize?: StateConversion<TStateModel, TPersistedState>;
}

// @public (undocumented)
export interface RecoverableStatePersistenceOptions {
keyValueStore?: KeyValueStore | PromiseLike<KeyValueStore>;
persistenceEnabled?: boolean;
persistenceTimeoutMillis?: number;
persistStateKey: string;
}

// @public (undocumented)
export type RegExpInput = RegExp | RegExpObject;

// @public (undocumented)
export interface RegExpObject {
// (undocumented)
regexp: RegExp;
}

// @public
class Request_2<UserData extends Dictionary = Dictionary> {
constructor(options: RequestOptions<UserData>);
get crawlDepth(): number;
set crawlDepth(value: number);
errorMessages: string[];
handledAt?: string;
headers?: Record<string, string>;
id?: string;
intoFetchAPIRequest(): Request;
get label(): string | undefined;
set label(value: string | undefined);
loadedUrl?: string;
get maxRetries(): number | undefined;
set maxRetries(value: number | undefined);
method: AllowedHttpMethods;
noRetry: boolean;
payload?: string;
pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
retryCount: number;
get sessionId(): string | undefined;
set sessionId(value: string | undefined);
get skipNavigation(): boolean;
set skipNavigation(value: boolean);
get skippedReason(): SkippedRequestReason | undefined;
set skippedReason(value: SkippedRequestReason | undefined);
get state(): RequestState;
set state(value: RequestState);
uniqueKey: string;
url: string;
userData: UserData;
}
export { Request_2 as Request }

// @public (undocumented)
export class RequestHandlerError extends Error {
constructor(error: unknown, options?: ErrorOptions);
}

// @public
export class RequestList implements IRequestLoader {
// (undocumented)
[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
// (undocumented)
fetchNextRequest(): Promise<Request_2 | null>;
// (undocumented)
getHandledCount(): Promise<number>;
getPendingCount(): Promise<number>;
getState(): RequestListState;
getTotalCount(): Promise<number>;
// (undocumented)
isEmpty(): Promise<boolean>;
// (undocumented)
isFinished(): Promise<boolean>;
// (undocumented)
markRequestAsHandled(request: Request_2): Promise<void>;
static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
// (undocumented)
persistState(): Promise<void>;
teardown(): Promise<void>;
toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public (undocumented)
export interface RequestListOptions {
httpClient?: BaseHttpClient;
keepDuplicateUrls?: boolean;
persistRequestsKey?: string;
persistStateKey?: string;
proxyConfiguration?: IProxyConfiguration;
sources?: RequestListSource[];
sourcesFunction?: RequestListSourcesFunction;
state?: RequestListState;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type RequestListSource = string | Source;

// @public (undocumented)
export type RequestListSourcesFunction = () => Promise<RequestListSource[]>;

// @public
export interface RequestListState {
inProgress: string[];
nextIndex: number;
nextUniqueKey: string | null;
}

// @public
export type RequestManagerOpener<T extends IRequestManager = IRequestManager> = (identifier: string | StorageIdentifier, options?: StorageOpenOptions) => Promise<T>;

// @public
export class RequestManagerTandem implements IRequestManager {
// (undocumented)
[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
constructor(requestLoader: IRequestLoader, requestManager: IRequestManager | (() => IRequestManager | Promise<IRequestManager>));
// (undocumented)
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
// (undocumented)
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
// (undocumented)
getHandledCount(): Promise<number>;
// (undocumented)
getPendingCount(): Promise<number>;
// (undocumented)
getTotalCount(): Promise<number>;
// (undocumented)
isEmpty(): Promise<boolean>;
// (undocumented)
isFinished(): Promise<boolean>;
// (undocumented)
markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
persistState(): Promise<void>;
purge(): Promise<void>;
// (undocumented)
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
}

// @public
export interface RequestOptions<UserData extends Dictionary = Dictionary> {
alwaysEnqueue?: boolean;
crawlDepth?: number;
headers?: Record<string, string>;
keepUrlFragment?: boolean;
label?: string;
maxRetries?: number;
method?: AllowedHttpMethods | Lowercase<AllowedHttpMethods>;
noRetry?: boolean;
payload?: string;
sessionId?: string;
skipNavigation?: boolean;
uniqueKey?: string;
url: string;
useExtendedUniqueKey?: boolean;
userData?: UserData;
}

// @public
export class RequestQueue implements IStorage, IRequestManager {
// (undocumented)
[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
addRequests(requestsLike: RequestsLike, options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
addRequestsBatched(requests: ReadonlyDeep<RequestsLike>, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
// (undocumented)
readonly backend: RequestQueueBackend;
drop(): Promise<void>;
fetchNextRequest<T extends Dictionary = Dictionary>(): Promise<Request_2<T> | null>;
getHandledCount(): Promise<number>;
getInfo(): Promise<RequestQueueInfo>;
getPendingCount(): Promise<number>;
getRequest<T extends Dictionary = Dictionary>(uniqueKey: string): Promise<Request_2<T> | null>;
getTotalCount(): Promise<number>;
// (undocumented)
readonly id: string;
isEmpty(): Promise<boolean>;
isFinished(): Promise<boolean>;
// (undocumented)
readonly log: CrawleeLogger;
markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | null>;
// (undocumented)
readonly name?: string;
static open(identifier?: string | StorageIdentifier | null, options?: StorageOpenOptions): Promise<RequestQueue>;
purge(): Promise<void>;
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
get stats(): RequestQueueStats;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
class RequestQueueBackend_2 extends BaseClient implements storage.RequestQueueBackend {
constructor(options: RequestQueueBackendOptions);
// (undocumented)
accessedAt: Date;
// (undocumented)
addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
cacheKey: string;
// (undocumented)
createdAt: Date;
// (undocumented)
drop(): Promise<void>;
// (undocumented)
fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
// (undocumented)
getMetadata(): Promise<storage.RequestQueueInfo>;
// (undocumented)
getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
// (undocumented)
handledRequestCount: number;
// (undocumented)
isEmpty(): Promise<boolean>;
// (undocumented)
isFinished(): Promise<boolean>;
listItems(): Promise<storage.UpdateRequestSchema[]>;
// (undocumented)
markRequestAsHandled(request: storage.UpdateRequestSchema): Promise<storage.QueueOperationInfo | undefined>;
// (undocumented)
modifiedAt: Date;
// (undocumented)
name?: string;
// (undocumented)
pendingRequestCount: number;
// (undocumented)
purge(): Promise<void>;
// (undocumented)
reclaimRequest(request: storage.UpdateRequestSchema, options?: storage.RequestQueueOperationOptions): Promise<storage.QueueOperationInfo | undefined>;
// (undocumented)
toRequestQueueInfo(): storage.RequestQueueInfo;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface RequestQueueBackendOptions {
cacheKey?: string;
// (undocumented)
id?: string;
// (undocumented)
name?: string;
// (undocumented)
storageBackend: MemoryStorageBackend;
}

// @public
export interface RequestQueueJournalEntry {
// (undocumented)
forefront: boolean;
// (undocumented)
requests: JournaledRequest[];
// (undocumented)
type: 'requestQueue';
writeThrough: boolean;
}

// @public (undocumented)
export interface RequestQueueOperationInfo extends QueueOperationInfo {
// (undocumented)
forefront: boolean;
// (undocumented)
uniqueKey: string;
}

// @public (undocumented)
export interface RequestQueueOperationOptions {
forefront?: boolean;
}

// @public (undocumented)
export interface RequestQueueOptions {
// (undocumented)
backend: RequestQueueBackend;
metadata: RequestQueueInfo;
proxyConfiguration?: IProxyConfiguration;
}

// @public
export interface RequestQueueStats {
headItemReadCount: number;
writeCount: number;
}

// @public (undocumented)
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];

// @public (undocumented)
export enum RequestState {
// (undocumented)
AFTER_NAV = 2,
// (undocumented)
BEFORE_NAV = 1,
// (undocumented)
DONE = 4,
// (undocumented)
ERROR = 6,
// (undocumented)
ERROR_HANDLER = 5,
// (undocumented)
REQUEST_HANDLER = 3,
// (undocumented)
SKIPPED = 7,
// (undocumented)
UNPROCESSED = 0
}

// @public
export class RequestThrottledError extends RetryRequestError {
constructor(message?: string);
}

// @public
export interface RequestTransform {
// (undocumented)
(original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
}

// @public
export class RequestValidationError extends NonRetryableError {
constructor(label: string | symbol, issues: readonly SchemaIssue[]);
// (undocumented)
readonly issues: readonly SchemaIssue[];
// (undocumented)
readonly label: string | symbol;
}

// @public (undocumented)
export type ResolvedConfigValues = FieldsOutput<typeof crawleeConfigFields>;

// @public
export function resolveStorageIdentifier(identifier: string | StorageIdentifier | null | undefined, storageBackend: StorageBackend, storageType: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<ExplicitStorageIdentifier>;

// @public (undocumented)
export interface ResponseLike {
// (undocumented)
headers?: Record<string, string | string[] | undefined> | (() => Record<string, string | string[] | undefined>);
// (undocumented)
url?: string | (() => string);
}

// @public (undocumented)
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
// (undocumented)
id: string;
log: CrawleeLogger;
proxyInfo?: ProxyInfo;
pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
request: Request_2<UserData>;
// (undocumented)
session: ISession;
useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
}

// @public
export class RetryRequestError extends Error {
constructor(message?: string);
}

// @public
export interface RouteOptions {
requestHandlerTimeoutSecs?: number;
}

// @public
export class Router<Context extends RestrictedCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
addDefaultHandler<UserData extends Dictionary = DefaultRouteUserData<Routes, GetUserDataFromRequest<Context['request']>>>(handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
addHandler<Label extends keyof Routes & string>(label: Label, handler: (ctx: RouterHandlerContext<Context, Routes[Label], Routes>) => Awaitable<void>, options?: RouteOptions): void;
addHandler<UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(label: RouterLabel<Routes>, handler: (ctx: RouterHandlerContext<Context, UserData, Routes>) => Awaitable<void>, options?: RouteOptions): void;
static create<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
// (undocumented)
static create<Context extends RestrictedCrawlingContext = CrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
// (undocumented)
static create<Context extends RestrictedCrawlingContext = CrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
getHandler(label?: string | symbol): (ctx: Context) => Awaitable<void>;
getMaxTimeoutSecs(): number | undefined;
getTimeoutSecs(label?: string | symbol): number | undefined;
use(middleware: (ctx: Context) => Awaitable<void>): void;
}

// @public (undocumented)
export interface RouterHandler<Context extends RestrictedCrawlingContext = CrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends Router<Context, Routes> {
// (undocumented)
(ctx: Context): Awaitable<void>;
}

// @public
export type RouterHandlerContext<Context, UserData extends Dictionary, Routes extends Record<keyof Routes, Dictionary>> = Omit<Context, 'request' | 'addRequests' | 'enqueueLinks'> & {
request: LoadedRequest<Request_2<UserData>>;
addRequests: TypedContextAddRequests<Routes>;
} & (Context extends {
enqueueLinks: infer EnqueueLinks;
} ? {
enqueueLinks: TypedContextEnqueueLinks<EnqueueLinks, Routes>;
} : {});

// @public
export type RouterLabel<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | symbol : (keyof Routes & string) | symbol;

// @public (undocumented)
export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary>> = {
[Label in keyof Routes]: (ctx: Omit<Context, 'request'> & {
request: Request_2<Routes[Label]>;
}) => Awaitable<void>;
};

// @public
export type RouteSchemas = Record<string, StandardSchemaV1> & {
[defaultRoute]?: StandardSchemaV1;
};

// @public
export type RoutesFromSchemas<Schemas extends RouteSchemas> = {
[Label in Extract<keyof Schemas, string>]: SchemaUserData<Schemas[Label]>;
} & (Schemas extends {
[defaultRoute]: StandardSchemaV1;
} ? {
[defaultRoute]: SchemaUserData<Schemas[typeof defaultRoute]>;
} : {});

// @public
export interface SchemaIssue {
// (undocumented)
readonly message: string;
// (undocumented)
readonly path?: readonly (PropertyKey | {
key: PropertyKey;
})[];
}

// Not exported by the entry point; reachable only as a referenced type.
// @public
type SchemaUserData<Schema extends StandardSchemaV1> = StandardSchemaV1.InferOutput<Schema> extends Dictionary ? StandardSchemaV1.InferOutput<Schema> : Dictionary;

// @public
export function serializeValue(value: unknown, contentType?: string): {
value: Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
contentType: string;
};

// @public
export class ServiceConflictError extends Error {
constructor(serviceName: string, newValue: unknown, existingValue: unknown);
}

// @public
export class ServiceLocator implements ServiceLocatorInterface {
constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
// (undocumented)
getChildLog(prefix: string): CrawleeLogger;
// (undocumented)
getConfiguration(): Configuration;
// (undocumented)
getEventManager(): EventManager;
// (undocumented)
getLogger(): CrawleeLogger;
// (undocumented)
getStorageBackend(): StorageBackend;
// (undocumented)
getStorageInstanceManager(): StorageInstanceManager;
// (undocumented)
reset(): void;
// (undocumented)
setConfiguration(configuration: Configuration): void;
// (undocumented)
setEventManager(eventManager: EventManager): void;
// (undocumented)
setLogger(logger: CrawleeLogger): void;
// (undocumented)
setStorageBackend(storageBackend: StorageBackend): void;
}

// @public (undocumented)
export const serviceLocator: ServiceLocatorInterface;

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface ServiceLocatorInterface {
getChildLog(prefix: string): CrawleeLogger;
getConfiguration(): Configuration;
getEventManager(): EventManager;
getLogger(): CrawleeLogger;
getStorageBackend(): StorageBackend;
getStorageInstanceManager(): StorageInstanceManager;
setConfiguration(configuration: Configuration): void;
setEventManager(eventManager: EventManager): void;
setLogger(logger: CrawleeLogger): void;
setStorageBackend(storageBackend: StorageBackend): void;
}

// @public
export class Session implements ISession {
constructor(options?: SessionOptions);
// (undocumented)
get cookieJar(): CookieJar;
// (undocumented)
get createdAt(): Date;
// (undocumented)
get errorScore(): number;
// (undocumented)
get errorScoreDecrement(): number;
// (undocumented)
get expiresAt(): Date;
// (undocumented)
get fingerprint(): SessionFingerprint | undefined;
set fingerprint(fingerprint: SessionFingerprint | undefined);
getCookieString(url: string): Promise<string>;
getState(): SessionState;
// (undocumented)
readonly id: string;
isBlocked(): boolean;
isExpired(): boolean;
isMaxUsageCountReached(): boolean;
isUsable(): boolean;
markBad(): void;
markGood(): void;
// (undocumented)
get maxErrorScore(): number;
// (undocumented)
get maxUsageCount(): number;
// (undocumented)
get proxyInfo(): ProxyInfo | undefined;
retire(): void;
get retired(): boolean;
setCookie(rawCookie: string, url: string): Promise<void>;
// (undocumented)
get usageCount(): number;
// (undocumented)
readonly userData: Dictionary;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];

// @public
export class SessionError extends Error {
constructor(message?: string);
}

// @public (undocumented)
export interface SessionOptions {
// (undocumented)
cookieJar?: CookieJar;
createdAt?: Date;
// (undocumented)
errorScore?: number;
errorScoreDecrement?: number;
expiresAt?: Date;
fingerprint?: SessionFingerprint;
id?: string;
// (undocumented)
log?: CrawleeLogger;
maxAgeSecs?: number;
maxErrorScore?: number;
maxUsageCount?: number;
// (undocumented)
proxyInfo?: ProxyInfo;
retired?: boolean;
usageCount?: number;
userData?: Dictionary;
}

// @public
export class SessionPool implements ISessionPool {
// (undocumented)
[Symbol.asyncDispose](): Promise<void>;
constructor(options?: SessionPoolOptions);
addSession(options?: Session | SessionOptions): Promise<void>;
getSession(sessionId?: string): Promise<Session | undefined>;
getState(): Promise<{
usableSessionsCount: number;
retiredSessionsCount: number;
sessions: SessionState[];
}>;
// (undocumented)
readonly id: string;
newSession(sessionOptions?: SessionOptions): Promise<Session>;
persistState(options?: PersistenceOptions): Promise<void>;
// (undocumented)
resetStore(options?: PersistenceOptions): Promise<void>;
retiredSessionsCount(): Promise<number>;
teardown(input?: {
persistState?: boolean;
}): Promise<void>;
usableSessionsCount(): Promise<number>;
}

// @public (undocumented)
export interface SessionPoolOptions {
createSessionFunction?: CreateSession;
id?: string | number;
maxPoolSize?: number;
persistenceOptions?: PersistenceOptions;
persistStateKey?: string;
persistStateKeyValueStoreId?: string;
sessionOptions?: SessionOptions;
sessionReuseStrategy?: SessionReuseStrategy;
}

// @public (undocumented)
export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];

// @public
export class SitemapRequestLoader implements IRequestLoader {
// (undocumented)
[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
// (undocumented)
fetchNextRequest(): Promise<Request_2 | null>;
// (undocumented)
getHandledCount(): Promise<number>;
// (undocumented)
getPendingCount(): Promise<number>;
// (undocumented)
getTotalCount(): Promise<number>;
// (undocumented)
isEmpty(): Promise<boolean>;
// (undocumented)
isFinished(): Promise<boolean>;
isSitemapFullyLoaded(): boolean;
// (undocumented)
markRequestAsHandled(request: Request_2): Promise<void>;
static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
// (undocumented)
persistState(): Promise<void>;
teardown(): Promise<void>;
toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
}

// @public (undocumented)
export interface SitemapRequestLoaderOptions extends UrlConstraints {
enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
httpClient?: BaseHttpClient;
maxBufferSize?: number;
parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
persistenceOptions?: {
enable?: boolean;
};
persistStateKey?: string;
proxyUrl?: string;
signal?: AbortSignal;
sitemapUrls: string[];
timeoutMillis?: number;
}

// @public (undocumented)
export type SkippedRequestCallback = (args: {
url: string;
reason: SkippedRequestReason;
}) => Awaitable<void>;

// @public (undocumented)
export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';

// @public (undocumented)
export interface SnapshotResult {
// (undocumented)
htmlFileName?: string;
// (undocumented)
screenshotFileName?: string;
}

// @public
export class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
clear(): void;
getAll(): T[];
getSample(sampleDurationMillis?: number): T[];
push(snapshot: T, now?: Date): void;
useSampleWindow(maxSampleWindowMillis: number): void;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface SnapshottableProperties {
// (undocumented)
body?: unknown;
// (undocumented)
page?: BrowserPage;
}

// @public (undocumented)
export type Source = (Partial<RequestOptions> & {
requestsFromUrl?: string;
regex?: RegExp;
}) | Request_2;

// @public
export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;

// @public
export class StateValidationError extends Error {
constructor(persistStateKey: string, issues: readonly SchemaIssue[]);
// (undocumented)
readonly issues: readonly SchemaIssue[];
// (undocumented)
readonly persistStateKey: string;
}

// @public
export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt' | 'crawlerStartedAt' | 'crawlerFinishedAt' | 'requestMinDurationMillis' | 'requestsFailedPerMinute' | 'requestsFinishedPerMinute' | 'requestRetryHistogram' | 'instanceStart'> {
// (undocumented)
crawlerFinishedAt: string | null;
crawlerLastStartTimestamp: number;
crawlerStartedAt: string | null;
// (undocumented)
requestAvgFailedDurationMillis: number | null;
// (undocumented)
requestAvgFinishedDurationMillis: number | null;
// (undocumented)
requestMinDurationMillis: number | null;
requestRetryHistogram: (number | null)[];
// (undocumented)
requestsFailedPerMinute: number | null;
// (undocumented)
requestsFinishedPerMinute: number | null;
// (undocumented)
requestsTotal: number;
// (undocumented)
requestTotalDurationMillis: number;
// (undocumented)
statsId: string;
// (undocumented)
statsPersistedAt: string;
}

// @public
export class Statistics<StateExtension extends object = {}, PersistedStateExtension extends object = StateExtension> implements IStatistics<StateExtension> {
constructor(options?: StatisticsOptions<StateExtension, PersistedStateExtension>);
calculate(): CalculatedStatistics;
readonly errorTracker: ErrorTracker;
readonly errorTrackerRetry: ErrorTracker;
readonly id: string;
persistState(): Promise<void>;
registerStatusCode(code: number): void;
get requestRetryHistogram(): number[];
reset(): void;
resetStore(): Promise<void>;
startCapturing(): Promise<void>;
get state(): StatisticState & StateExtension;
stopCapturing(): Promise<void>;
toJSON(): StatisticPersistedState & PersistedStateExtension;
}

// @public
export interface StatisticsOptions<StateExtension extends object = {}, PersistedStateExtension extends object = StateExtension> {
id?: string;
keyValueStore?: KeyValueStore;
log?: CrawleeLogger;
logIntervalSecs?: number;
logMessage?: string;
persistenceOptions?: PersistenceOptions;
saveErrorSnapshots?: boolean;
stateExtension?: StatisticStateExtensionOptions<StateExtension, PersistedStateExtension>;
}

// @public
export interface StatisticState {
// (undocumented)
crawlerFinishedAt: Date | string | null;
// (undocumented)
crawlerRuntimeMillis: number;
// (undocumented)
crawlerStartedAt: Date | string | null;
// (undocumented)
errors: Record<string, unknown>;
instanceStart: number;
// (undocumented)
requestMaxDurationMillis: number;
// (undocumented)
requestMinDurationMillis: number;
requestRetryHistogram: number[];
// (undocumented)
requestsFailed: number;
// (undocumented)
requestsFailedPerMinute: number;
// (undocumented)
requestsFinished: number;
// (undocumented)
requestsFinishedPerMinute: number;
// (undocumented)
requestsRetries: number;
// (undocumented)
requestsWithStatusCode: Record<string, number>;
// (undocumented)
requestTotalFailedDurationMillis: number;
// (undocumented)
requestTotalFinishedDurationMillis: number;
// (undocumented)
retryErrors: Record<string, unknown>;
// (undocumented)
statsPersistedAt: Date | string | null;
}

// @public
export interface StatisticStateExtensionOptions<StateExtension extends object, PersistedStateExtension extends object = StateExtension> {
defaultState?: StateExtension | (() => StateExtension);
deserialize?: SyncStateConversion<unknown, StateExtension>;
serialize?: SyncStateConversion<StateExtension, PersistedStateExtension>;
}

export { StorageBackend }

// @public
export class StorageBackendLoadSignal implements LoadSignal {
constructor(options?: StorageBackendLoadSignalOptions);
// (undocumented)
getSample(sampleDurationMillis?: number): LoadSnapshot[];
// (undocumented)
readonly name = "storageBackendInfo";
// (undocumented)
readonly overloadedRatio: number;
// (undocumented)
start(context: LoadSignalStartContext): Promise<void>;
// (undocumented)
stop(): Promise<void>;
}

// @public
export interface StorageBackendLoadSignalOptions {
maxErrors?: number;
overloadedRatio?: number;
snapshotIntervalSecs?: number;
}

export { StorageIdentifier }

// @public
export class StorageInstanceManager {
clearCache(): void;
openStorage<TStorage extends IStorage>(cls: Constructor<TStorage>, input: (ExplicitStorageIdentifier | DefaultStorageIdentifier) & {
backendOpener: () => Promise<DatasetBackend | KeyValueStoreBackend | RequestQueueBackend>;
backendCacheKey: Hashable;
}): Promise<TStorage>;
removeFromCache(instance: IStorage): void;
}

// @public
export interface StorageOpenOptions {
configuration?: Configuration;
httpClient?: BaseHttpClient;
proxyConfiguration?: IProxyConfiguration;
storageBackend?: StorageBackend;
}

// @public
export class StorageStatsTracker<T extends Record<keyof T, number>> {
constructor(initial: T);
add(key: keyof T, by?: number): void;
get current(): T;
}

// @public
export class StorageTransaction implements StorageTransactionView {
commit(): Promise<void>;
// (undocumented)
get datasetItems(): {
item: Dictionary;
datasetId: string;
}[];
dispose(): void;
// (undocumented)
get enqueuedUrls(): {
url: string;
label?: string;
}[];
get isActive(): boolean;
readonly journal: JournalEntry[];
// (undocumented)
get keyValueStoreChanges(): Record<string, Record<string, {
changedValue: unknown;
options?: RecordOptions;
}>>;
readonly policy: StorageWritePolicy;
rollback(): void;
run<T>(callback: () => Awaitable<T>): Promise<T>;
// (undocumented)
get state(): StorageTransactionState;
}

// @public (undocumented)
export interface StorageTransactionOptions {
commitTimeoutMillis?: number;
policy?: Partial<StorageWritePolicy>;
}

// @public (undocumented)
export type StorageTransactionState = 'open' | 'committing' | 'committed' | 'failed' | 'rolledBack';

// @public
export interface StorageTransactionView {
readonly datasetItems: {
item: Dictionary;
datasetId: string;
}[];
readonly enqueuedUrls: {
url: string;
label?: string;
}[];
readonly keyValueStoreChanges: Record<string, Record<string, {
changedValue: unknown;
options?: RecordOptions;
}>>;
// (undocumented)
readonly state: StorageTransactionState;
}

// @public
export type StorageWriteMode = 'deferred' | 'writeThrough';

// @public
export interface StorageWritePolicy {
requestQueue: StorageWriteMode;
}

// @public
export interface SupportsDomainThrottling {
// (undocumented)
assertNoStalledDomains(): Promise<void>;
// (undocumented)
recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
// (undocumented)
setCrawlDelay(url: string, delaySeconds: number): boolean;
}

// @public
export function supportsDomainThrottling(manager: unknown): manager is SupportsDomainThrottling;

// @public
export type SyncStateConversion<TFrom, TTo> = ((value: TFrom) => TTo) | StandardSchemaV1<TFrom, TTo>;

// @public
export interface SystemInfo {
// (undocumented)
cpuInfo: LoadSignalInfo;
// (undocumented)
eventLoopInfo: LoadSignalInfo;
isSystemIdle: boolean;
loadSignalInfo?: Record<string, LoadSignalInfo>;
// (undocumented)
memCurrentBytes?: number;
// (undocumented)
memInfo: LoadSignalInfo;
// (undocumented)
memTotalBytes?: number;
// (undocumented)
storageBackendInfo: LoadSignalInfo;
}

// @public
export interface TaskLoopPredicates {
isFinishedFunction?: () => Promise<boolean>;
isTaskReadyFunction?: () => Promise<boolean>;
}

// @public
export class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager, SupportsDomainThrottling {
// (undocumented)
[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;
constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
// (undocumented)
addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
assertNoStalledDomains(): Promise<void>;
// (undocumented)
drop(): Promise<void>;
fetchNextRequest<R extends Dictionary = Dictionary>(): Promise<Request_2<R> | null>;
// (undocumented)
getHandledCount(): Promise<number>;
// (undocumented)
getPendingCount(): Promise<number>;
// (undocumented)
getTotalCount(): Promise<number>;
get innerManager(): T;
isEmpty(): Promise<boolean>;
isFinished(): Promise<boolean>;
// (undocumented)
markRequestAsHandled(request: Request_2): Promise<RequestQueueOperationInfo | void | null>;
// (undocumented)
persistState(): Promise<void>;
purge(): Promise<void>;
purgeDomainQueues(): Promise<void>;
// (undocumented)
reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
recordDomainDelay(url: string, retryAfterMs?: number | null): boolean;
setCrawlDelay(url: string, delaySeconds: number): boolean;
// (undocumented)
setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
}

// @public
export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRequestManager> {
baseDelaySecs?: number;
domains: string[] | 'all';
inner: T;
maxDelaySecs?: number;
maxDomainStallSecs?: number;
maxThrottledDomains?: number;
minCrawlDelaySecs?: number;
persistStateKey?: string;
requestManagerOpener?: RequestManagerOpener<T>;
throttleBy?: 'hostname' | 'registrableDomain';
}

// @public
export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;

// @public
export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;

// Not exported by the entry point; reachable only as a referenced type.
// @public
type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
[Label in keyof Routes & string]: {
label: Label;
userData?: Routes[Label];
};
}[keyof Routes & string] | {
label?: undefined;
userData?: Dictionary;
});

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface UrlConstraints {
exclude?: readonly UrlPatternInput[];
include?: readonly UrlPatternInput[];
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type UrlList = (string | null)[];

// @public
export type UrlPatternInput = GlobInput | RegExpInput;

// @public (undocumented)
export interface UrlPatternObject {
// (undocumented)
glob?: string;
// (undocumented)
regexp?: RegExp;
}

// @public
export function useState<State extends Dictionary = Dictionary>(name?: string, defaultValue?: State, options?: UseStateOptions): Promise<State>;

// @public (undocumented)
export interface UseStateOptions {
// (undocumented)
configuration?: Configuration;
keyValueStoreName?: string | null;
}

// @public
export function withDirectStorageAccess<T>(callback: () => Awaitable<T>): Promise<T>;

// @public (undocumented)
export type WithRequired<T, K extends keyof T> = T & {
[P in K]-?: T[P];
};

// @public
export function withStorageTransaction<T>(callback: (transaction: StorageTransaction) => Awaitable<T>, options?: StorageTransactionOptions): Promise<T>;

// (No @packageDocumentation comment for this package)