Scraper Studio functions reference

navigate(input.url);
let {created} = job;
console.log('current url', location.href);

navigate(input.url);
navigate('https://example.com');
navigate('https://example.com', {wait_until: 'domcontentloaded'});
navigate('https://example.com', {referer: 'https://google.com'});
navigate('https://example.com', {timeout: 45000});
navigate('https://example.com', {allow_status: [404]});
navigate('https://example.com', {
  fingerprint: {screen: {width: 400, height: 400}},
});

let res = request('https://www.example.com');
let res = request({
  url: 'https://www.example.com',
  method: 'POST',
  headers: {'Content-type': 'application/json'},
  body: {hello: 'world'},
});

next_stage({url: 'https://example.com', page: 1});

run_stage(2, {url: 'https://example.com', page: 1});

rerun_stage({url: 'https://example.com/other-page'});

let {pages} = load_sitemap({url: 'https://example.com/sitemap.xml.gz'});
let {children} = load_sitemap({url: 'https://example.com/sitemap-index.xml'});

let {href} = parse().anchor_elem_data;
collect({final_url: resolve_url(href)});

navigate('http://google.com');
let redirects = redirect_history();
// ['http://google.com', 'http://www.google.com', 'https://www.google.com/']

let headers = response_headers();
console.log('content-type', headers['content-type']);

collect({status_code: status_code()});

wait('#welcome-splash');
wait('.search-results .product');
wait('[href^="/product"]');
wait('#welcome-splash', {timeout: 5000});
wait('#welcome-splash', {hidden: true});
wait('#welcome-splash', {inside: '#iframe_id'});

wait_any(['#title', '#notfound']);

wait_visible('#welcome-splash');
wait_visible('#welcome-splash', {timeout: 5000});

wait_hidden('#welcome-splash');
wait_hidden('#welcome-splash', {timeout: 5000});

wait_for_text('.location', 'New York');

wait_for_parser_value('profile');
wait_for_parser_value('listings.0.price', v => parseInt(v) > 0, {timeout: 5000});

wait_network_idle();
wait_network_idle({
  timeout: 1e3,
  ignore: [/long_request/, 'https://example.com'],
});

wait_page_idle();
wait_page_idle({
  ignore: ['.live-clock', '.carousel'],
  idle_timeout: 1000,
});

click('#show-more');
$('#show-more').click();

// Click the map pin closest to the center of a map
let box = bounding_box('#map');
let center = {x: (box.left + box.right) / 2, y: (box.top + box.bottom) / 2};
click('.map-pin', {coordinates: center});

right_click('#item');

hover('#item');

mouse_to(0, 0);

type('#location', 'New York');
type('#location', 'New York', {replace: true});
type('[id$=input-box]', 'search term');
type('#search', ['Some text', 'Enter']);
type('#search', ['Backspace']);

press_key('Enter');
press_key('Backspace');

select('#country', 'Canada');

scroll_to('.author-profile');
scroll_to('top');
scroll_to('bottom');
scroll_to('top', {immediate: true});

scroll_to_all('.author-profiles');

load_more('.search-results');
load_more('.search-results', {
  children: '.result-item',
  trigger_selector: '.btn-load-more',
  timeout: 10000,
});

close_popup('.popup', '.popup_close');
close_popup('iframe.with-popup', '.popup_close', {click_inside: 'iframe.with-popup'});

solve_captcha();
solve_captcha({type: 'simple', selector: '#image', input: '#input'});

let box = bounding_box('.product-list');
// box == {top, right, bottom, left, x, y, width, height}

el_exists('#example');            // true
el_exists('.does_not_exist');     // false
el_exists('.does_not_exist', 5e3); // false after 5 seconds

el_is_visible('#example');
el_is_visible('.is_not_visible', 5e3);

track_event_listeners();

disable_event_listeners();
disable_event_listeners(['hover', 'click']);

freeze_page();

tag_response('resp', /url/, {jsonp: true});
tag_response('resp', /url/, {allow_error: true});

tag_response('resp', (req, res) => {
  if (req.url.includes('/api/')) {
    return {
      request_body: req.body,
      request_headers: req.headers,
      response_body: res.body,
      response_headers: res.headers,
    };
  }
});

tag_response('teams', /\/api\/teams/);
navigate('https://example.com/sports');
let teams = parse().teams;
for (let team of teams)
  collect(team);

tag_all_responses('profiles', /\/api\/profile/);
navigate('https://example.com/sports');
let profiles = parse().profiles;
for (let profile of profiles)
  collect(profile);

tag_script('ssr_state', '#__SSR_DATA__');
navigate('https://example.com/');
collect(parse().ssr_state);

tag_window_field('initData', '__INIT_DATA__');

tag_image('image', '#product-image');

tag_video('video', '#product-video', {download: true});

tag_screenshot('html_screenshot', {filename: 'screen'});
tag_screenshot('view', {full_page: false});

let SEC = 1000;
let download = tag_download(/example.com\/foo\/bar/);
click('button#download');
let file1 = download.next_file({timeout: 10 * SEC});
let file2 = download.next_file({timeout: 20 * SEC});
collect({file1, file2});

tag_serp('serp_bing_results', 'bing');
tag_serp('serp_google_results', 'google');

let q = capture_graphql({
  payload: {id: 'ProfileQuery'},
});
navigate('https://example.com');

let [first_query, first_response] = q.wait_captured();
collect(first_response.data.profile);

let second = q.replay({
  variables: {other_id: 2},
});
collect(second.data.profile);

let page_data = parse();
collect({
  title: page_data.title,
  price: page_data.price,
});

collect({price: data.price});
collect(product, p => {
  if (!p.title)
    throw new Error('Product is missing a title');
});

set_lines(products_so_far);
set_lines(products_so_far, p => {
  if (!p.price)
    throw new Error('Missing price');
});

let $$ = load_html('<p id="p1">p1</p><p id="p2">p2</p>');
collect({data: $$('#p2').text()});

bad_input();
bad_input('Missing search term');

blocked();
blocked('Login page was shown');

dead_page();
dead_page('Product was removed');

detect_block({selector: '.foo'}, {exists: true});
detect_block({selector: '.bar'}, {has_text: 'text'});
detect_block({selector: '.baz'}, {has_text: /regex_pattern/});

country('us');

proxy_location({country: 'us'});
proxy_location({lat: 37.7749, long: 122.4194});
proxy_location({lat: 37.7749, long: 122.4194, country: 'US', radius: 100});

preserve_proxy_session();

set_session_cookie('example.com', 'session_id', 'abc123');

set_session_headers({'X-Custom-Header': 'value'});

let size = browser_size();
console.log(size.width, size.height);

emulate_device('iPhone X');
emulate_device('Pixel 2');

font_exists('Liberation Mono');

html_capture_options({
  coordinate_attributes: true,
});

embed_html_comment('trace-id: asdf123');

console.log(1, 'brightdata', [1, 2], {key: 'value'});
console.error('something went wrong');

verify_requests(({url, error, type, response}) => {
  if (response.status != 404 && type == 'Font')
    throw new Error('Font failed to load');
});

let img = new Image('https://example.com/image.png');
let vid = new Video('https://example.com/video.mp4');
let price = new Money(10, 'USD');

collect({image: img, video: vid, product_price: price});

let u = new URL('https://example.com');

let url = input.url;
let current_url = location.href;
$('#example').text();

let name = $('a').text_sane();                   // "foo bar baz"
let raw  = $('a').text();                        // "foo   bar\n\n\t baz"

$('.selector').filter_includes('text').click();

let img = new Image('https://example.com/image.png');
let price = new Money(10, 'USD');
collect({image: img, product_price: price});

click(['body', 'my-shadow-host', 'button.submit']);