crawlee-utils.api

Public API Report File for "@crawlee/utils"

Do not edit this file. It is a report generated by API Extractor.

import type { BaseHttpClient } from '@crawlee/http-client';
import type { CheerioAPI } from 'cheerio';
import type { CrawleeLogger } from '@crawlee/types';
import type { Dictionary } from '@crawlee/types';
import type { z } from 'zod';

// @public
export class ArgumentValidationError extends Error {
    constructor(error: z.ZodError, value: unknown, label?: string);
    readonly cause: z.ZodError;
    readonly issues: z.ZodError['issues'];
}

// @public
const DISCORD_REGEX: RegExp;

// @public
const DISCORD_REGEX_GLOBAL: RegExp;

// @public
export function discoverValidSitemaps(urls: string[], options?: {
    proxyUrl?: string;
    timeoutMillis?: number;
    signal?: AbortSignal;
    requestTimeoutMillis?: number;
    httpClient?: BaseHttpClient;
    logger?: CrawleeLogger;
}): AsyncIterable<string>;

// @public
export function downloadListOfUrls(options: DownloadListOfUrlsOptions): Promise<string[]>;

// @public (undocumented)
export interface DownloadListOfUrlsOptions {
    encoding?: BufferEncoding;
    httpClient?: BaseHttpClient;
    proxyUrl?: string;
    url: string;
    urlRegExp?: RegExp;
}

// @public
const EMAIL_REGEX: RegExp;

// @public
const EMAIL_REGEX_GLOBAL: RegExp;

// @public
function emailsFromText(text: string): string[];

// @public
function emailsFromUrls(urls: string[]): string[];

// @public
export enum EnqueueStrategy {
    All = "all",
    SameDomain = "same-domain",
    SameHostname = "same-hostname",
    SameOrigin = "same-origin"
}

// @public
export function extractUrls(options: ExtractUrlsOptions): string[];

// @public (undocumented)
export interface ExtractUrlsOptions {
    string: string;
    urlRegExp?: RegExp;
}

// @public
const FACEBOOK_REGEX: RegExp;

// @public
const FACEBOOK_REGEX_GLOBAL: RegExp;

// @public
export function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;

// @public
const INSTAGRAM_REGEX: RegExp;

// @public
const INSTAGRAM_REGEX_GLOBAL: RegExp;

// @public
const LINKEDIN_REGEX: RegExp;

// @public
const LINKEDIN_REGEX_GLOBAL: RegExp;

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface NestedSitemap {
    // (undocumented)
    loc: string;
    // (undocumented)
    originSitemapUrl: null;
}

// @public (undocumented)
export interface OpenGraphProperty {
    // (undocumented)
    children: OpenGraphProperty[];
    // (undocumented)
    name: string;
    // (undocumented)
    outputName: string;
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type OpenGraphResult = string | string[] | Dictionary<string | Dictionary>;

// @public
function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;

// @public
export function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;

// @public (undocumented)
export function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;

// @public (undocumented)
export function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;

// @public (undocumented)
export interface ParseSitemapOptions {
    emitNestedSitemaps?: true | false;
    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
    httpClient?: BaseHttpClient;
    logger?: CrawleeLogger;
    maxDepth?: number;
    nestedSitemapFilter?: (sitemapUrl: string) => boolean;
    reportNetworkErrors?: boolean;
    sitemapRetries?: number;
    timeoutMillis?: number;
}

// @public
function phonesFromText(text: string): string[];

// @public
function phonesFromUrls(urls: string[]): string[];

// @public
const PINTEREST_REGEX: RegExp;

// @public
const PINTEREST_REGEX_GLOBAL: RegExp;

// @public
export class RobotsTxtFile {
    static find(url: string, options?: {
        signal?: AbortSignal;
        timeoutMillis?: number;
        proxyUrl?: string;
        httpClient?: BaseHttpClient;
        logger?: CrawleeLogger;
    }): Promise<RobotsTxtFile>;
    static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
    getCrawlDelay(userAgent?: string): number | undefined;
    getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
    isAllowed(url: string, userAgent?: string): boolean;
    parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
    parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
}

// @public (undocumented)
export interface RobotsTxtFileSitemapsOptions {
    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
}

// @public
export class Sitemap {
    constructor(urls: string[]);
    static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
    static load(urls: string | string[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
    static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
    // (undocumented)
    readonly urls: string[];
}

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
type SitemapSource = ({
    type: 'url';
    url: string;
} | {
    type: 'raw';
    content: string;
}) & {
    depth?: number;
};

// @public (undocumented)
export type SitemapUrl = SitemapUrlData & {
    originSitemapUrl: string;
};

// Not exported by the entry point; reachable only as a referenced type.
// @public (undocumented)
interface SitemapUrlData {
    // (undocumented)
    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
    // (undocumented)
    lastmod?: Date;
    // (undocumented)
    loc: string;
    // (undocumented)
    priority?: number;
}

// @public
export function sleep(millis?: number): Promise<void>;

declare namespace social {
    export {
        emailsFromText,
        emailsFromUrls,
        phonesFromText,
        phonesFromUrls,
        parseHandlesFromHtml,
        EMAIL_REGEX,
        EMAIL_REGEX_GLOBAL,
        SocialHandles,
        LINKEDIN_REGEX,
        LINKEDIN_REGEX_GLOBAL,
        INSTAGRAM_REGEX,
        INSTAGRAM_REGEX_GLOBAL,
        TWITTER_REGEX,
        TWITTER_REGEX_GLOBAL,
        FACEBOOK_REGEX,
        FACEBOOK_REGEX_GLOBAL,
        YOUTUBE_REGEX,
        YOUTUBE_REGEX_GLOBAL,
        TIKTOK_REGEX,
        TIKTOK_REGEX_GLOBAL,
        PINTEREST_REGEX,
        PINTEREST_REGEX_GLOBAL,
        DISCORD_REGEX,
        DISCORD_REGEX_GLOBAL
    }
}

// @public
interface SocialHandles {
    // (undocumented)
    discords: string[];
    // (undocumented)
    emails: string[];
    // (undocumented)
    facebooks: string[];
    // (undocumented)
    instagrams: string[];
    // (undocumented)
    linkedIns: string[];
    // (undocumented)
    phones: string[];
    // (undocumented)
    phonesUncertain: string[];
    // (undocumented)
    pinterests: string[];
    // (undocumented)
    tiktoks: string[];
    // (undocumented)
    twitters: string[];
    // (undocumented)
    youtubes: string[];
}

// @public
const TIKTOK_REGEX: RegExp;

// @public
const TIKTOK_REGEX_GLOBAL: RegExp;

// @public
const TWITTER_REGEX: RegExp;

// @public
const TWITTER_REGEX_GLOBAL: RegExp;

// @public
const YOUTUBE_REGEX: RegExp;

// @public
const YOUTUBE_REGEX_GLOBAL: RegExp;

// (No @packageDocumentation comment for this package)