rust 1,859,748 · typescript 235,215 · javascript 132,950 · python 20,426 · 6,405 files · digest 9ba5c926519d · built in 1.7 s · crystals off
crates/cre8-app-provider/src/adapters/legal/crawl.rs
Items
const USER_AGENT const ROBOTS_TTL const CRAWL_CURSOR_SCHEMA struct TransportError impl TransportError trait Transport struct NetworkTransport impl NetworkTransport impl Transport for NetworkTransport struct CrawlPolicy impl Default for CrawlPolicy struct Fetched struct PoliteFetcher impl PoliteFetcher static SLOTS static ROBOTS struct CachedArtifact const ARTIFACT_CACHE_ENTRIES const ARTIFACT_CACHE_TTL static CACHE struct CrawlCursor impl CrawlCursor static SLEPT_MS struct ScriptedTransport impl ScriptedTransport impl Transport for ScriptedTransport
Functions
By call-graph weight (PageRank; 1.0 is the average function). 43 functions in all.
| function | kind | lines | cyclomatic | callers | callees | weight | crystal |
|---|---|---|---|---|---|---|---|
| CrawlPolicy::defaultsrc/adapters/legal/crawl.rs:129 | trait impl | 9 | 1 | 10 | 0 | 1.5 | |
| PoliteFetcher::newsrc/adapters/legal/crawl.rs:156 | method | 3 | 1 | 10 | 0 | 1.5 | |
| PoliteFetcher::transportsrc/adapters/legal/crawl.rs:177 | method | 3 | 1 | 4 | 0 | 1.4 | |
| fetch_page_textsrc/adapters/legal/crawl.rs:367 | fn | 4 | 1 | 3 | 3 | 1.2 | |
| CrawlCursor::currentsrc/adapters/legal/crawl.rs:433 | method | 3 | 1 | 3 | 0 | 1.2 | |
| PoliteFetcher::get_with_headerssrc/adapters/legal/crawl.rs:188 | method | 5 | 1 | 3 | 3 | 1.1 | |
| fetch_artifactsrc/adapters/legal/crawl.rs:341 | fn | 24 | 3 | 3 | 7 | 1 | |
| PoliteFetcher::wait_for_hostsrc/adapters/legal/crawl.rs:270 | method | 12 | 2 | 3 | 2 | 1 | |
| PoliteFetcher::permittedsrc/adapters/legal/crawl.rs:206 | method | 25 | 6 | 2 | 5 | 1 | |
| host_slotssrc/adapters/legal/crawl.rs:310 | fn | 4 | 1 | 1 | 0 | 0.9 | |
| TransportError::is_retryablesrc/adapters/legal/crawl.rs:50 | method | 6 | 3 | 2 | 0 | 0.9 | |
| NetworkTransport::pagessrc/adapters/legal/crawl.rs:98 | method | 3 | 1 | 2 | 1 | 0.9 | |
| status_from_error_messagesrc/adapters/legal/crawl.rs:65 | fn | 5 | 1 | 2 | 0 | 0.8 | |
| PoliteFetcher::headerssrc/adapters/legal/crawl.rs:232 | method | 3 | 1 | 2 | 0 | 0.8 | |
| PoliteFetcher::get_with_retriessrc/adapters/legal/crawl.rs:236 | method | 31 | 5 | 1 | 3 | 0.8 | |
| NetworkTransport::newsrc/adapters/legal/crawl.rs:93 | method | 3 | 1 | 4 | 0 | 0.8 | |
| PoliteFetcher::with_user_agentsrc/adapters/legal/crawl.rs:162 | method | 4 | 1 | 4 | 0 | 0.8 | |
| CrawlCursor::startsrc/adapters/legal/crawl.rs:400 | method | 12 | 1 | 2 | 0 | 0.8 | |
| CrawlCursor::decodesrc/adapters/legal/crawl.rs:418 | method | 14 | 4 | 2 | 0 | 0.8 | |
| CrawlCursor::pin_fingerprintsrc/adapters/legal/crawl.rs:470 | method | 13 | 4 | 5 | 1 | 0.7 | |
| CrawlCursor::advancesrc/adapters/legal/crawl.rs:486 | method | 13 | 3 | 5 | 0 | 0.7 | |
| PoliteFetcher::with_sleepsrc/adapters/legal/crawl.rs:168 | method | 4 | 1 | 3 | 0 | 0.7 | |
| record_sleepsrc/adapters/legal/crawl.rs:545 | fn | 3 | 1 | 3 | 0 | 0.7 | |
| ScriptedTransport::newsrc/adapters/legal/crawl.rs:556 | method | 6 | 1 | 3 | 0 | 0.7 | |
| statussrc/adapters/legal/crawl.rs:573 | fn | 3 | 1 | 3 | 0 | 0.7 | |
| CrawlCursor::coverssrc/adapters/legal/crawl.rs:438 | method | 7 | 1 | 1 | 0 | 0.7 | |
| PoliteFetcher::robots_forsrc/adapters/legal/crawl.rs:283 | method | 25 | 6 | 1 | 7 | 0.7 | |
| CrawlCursor::resume_or_startsrc/adapters/legal/crawl.rs:447 | method | 20 | 3 | 4 | 3 | 0.6 | |
| NetworkTransport::artifactssrc/adapters/legal/crawl.rs:103 | method | 3 | 1 | 1 | 1 | 0.6 | |
| artifact_cachesrc/adapters/legal/crawl.rs:335 | fn | 4 | 1 | 1 | 0 | 0.6 | |
| robots_cachesrc/adapters/legal/crawl.rs:315 | fn | 4 | 1 | 1 | 0 | 0.6 | |
| PoliteFetcher::admitsrc/adapters/legal/crawl.rs:198 | method | 5 | 1 | 1 | 2 | 0.5 | |
| NetworkTransport::getsrc/adapters/legal/crawl.rs:109 | trait impl | 6 | 1 | 0 | 3 | 0.5 | |
| PoliteFetcher::policysrc/adapters/legal/crawl.rs:173 | method | 3 | 1 | 0 | 0 | 0.5 | |
| PoliteFetcher::getsrc/adapters/legal/crawl.rs:182 | method | 3 | 1 | 0 | 1 | 0.5 | |
| CrawlCursor::encodesrc/adapters/legal/crawl.rs:413 | method | 3 | 1 | 0 | 0 | 0.5 | |
| CrawlCursor::is_completesrc/adapters/legal/crawl.rs:500 | method | 3 | 1 | 0 | 0 | 0.5 | |
| status_is_read_from_network_client_errorssrc/adapters/legal/crawl.rs:510 | fn | 7 | 1 | 0 | 2 | 0.5 | |
| cursor_round_trips_and_refuses_other_scopes_and_changed_artifactssrc/adapters/legal/crawl.rs:519 | fn | 17 | 1 | 0 | 5 | 0.5 | |
| ScriptedTransport::getsrc/adapters/legal/crawl.rs:565 | trait impl | 6 | 1 | 0 | 0 | 0.5 | |
| fetcher_retries_server_errors_then_succeeds_and_respects_robotssrc/adapters/legal/crawl.rs:578 | fn | 14 | 1 | 0 | 6 | 0.5 | |
| fetcher_gives_up_on_final_statuses_and_refuses_disallowed_pathssrc/adapters/legal/crawl.rs:594 | fn | 13 | 1 | 0 | 7 | 0.5 | |
| unreachable_robots_disallows_the_crawlsrc/adapters/legal/crawl.rs:609 | fn | 6 | 1 | 0 | 6 | 0.5 |
Source not readable from this process.