Data Science Jupyter Notebooks

• Find the first occurrence of a tag.

first_link = soup.find('a')

• Find all occurrences of a tag.

all_links = soup.find_all('a')

• Find tags by their CSS class.

articles = soup.find_all('div', class_='article-content')

• Find a tag by its ID.

main_content = soup.find(id='main-container')

• Find tags by other attributes.

images = soup.find_all('img', attrs={'data-src': True})

• Find using a list of multiple tags.

headings = soup.find_all(['h1', 'h2', 'h3'])

• Find using a regular expression.

import re
links_with_blog = soup.find_all('a', href=re.compile(r'blog'))

• Find using a custom function.

# Finds tags with a 'class' but no 'id'
tags = soup.find_all(lambda tag: tag.has_attr('class') and not tag.has_attr('id'))

• Limit the number of results.

first_five_links = soup.find_all('a', limit=5)

• Use CSS Selectors to find one element.

footer = soup.select_one('#footer > p')

• Use CSS Selectors to find all matching elements.

article_links = soup.select('div.article a')

• Select direct children using CSS selector.

nav_items = soup.select('ul.nav > li')

IV. Extracting Data with BeautifulSoup

• Get the text content from a tag.

title_text = soup.title.get_text()

• Get stripped text content.

link_text = soup.find('a').get_text(strip=True)

• Get all text from the entire document.

all_text = soup.get_text()

• Get an attribute's value (like a URL).

link_url = soup.find('a')['href']

• Get the tag's name.

tag_name = soup.find('h1').name

• Get all attributes of a tag as a dictionary.

attrs_dict = soup.find('img').attrs

V. Parsing with lxml and XPath

• Import the library.

from lxml import html

• Parse HTML content with lxml.

tree = html.fromstring(response.content)

• Select elements using an XPath expression.

# Selects all <a> tags inside <div> tags with class 'nav'
links = tree.xpath('//div[@class="nav"]/a')

• Select text content directly with XPath.

# Gets the text of all <h1> tags
h1_texts = tree.xpath('//h1/text()')

• Select an attribute value with XPath.

# Gets all href attributes from <a> tags
hrefs = tree.xpath('//a/@href')

VI. Handling Dynamic Content (Selenium)

• Import the webdriver.

from selenium import webdriver

• Initialize a browser driver.

driver = webdriver.Chrome() # Requires chromedriver

• Navigate to a webpage.

driver.get('http://example.com')

• Find an element by its ID.

element = driver.find_element('id', 'my-element-id')

• Find elements by CSS Selector.

elements = driver.find_elements('css selector', 'div.item')

• Find an element by XPath.

button = driver.find_element('xpath', '//button[@type="submit"]')

• Click a button.

button.click()

• Enter text into an input field.

search_box = driver.find_element('name', 'q')
search_box.send_keys('Python Selenium')

• Wait for an element to become visible.

www.tgoop.com/DataScienceN/864

164 viewsNov 4 at 19:45

• Find the first occurrence of a tag.

first_link = soup.find('a')

all_links = soup.find_all('a')

articles = soup.find_all('div', class_='article-content')

main_content = soup.find(id='main-container')

images = soup.find_all('img', attrs={'data-src': True})

headings = soup.find_all(['h1', 'h2', 'h3'])

import re
links_with_blog = soup.find_all('a', href=re.compile(r'blog'))

# Finds tags with a 'class' but no 'id'
tags = soup.find_all(lambda tag: tag.has_attr('class') and not tag.has_attr('id'))

first_five_links = soup.find_all('a', limit=5)

footer = soup.select_one('#footer > p')

article_links = soup.select('div.article a')

nav_items = soup.select('ul.nav > li')

title_text = soup.title.get_text()

link_text = soup.find('a').get_text(strip=True)

all_text = soup.get_text()

link_url = soup.find('a')['href']

tag_name = soup.find('h1').name

attrs_dict = soup.find('img').attrs

from lxml import html

tree = html.fromstring(response.content)

# Selects all <a> tags inside <div> tags with class 'nav'
links = tree.xpath('//div[@class="nav"]/a')

# Gets the text of all <h1> tags
h1_texts = tree.xpath('//h1/text()')

# Gets all href attributes from <a> tags
hrefs = tree.xpath('//a/@href')

from selenium import webdriver

driver = webdriver.Chrome() # Requires chromedriver

driver.get('http://example.com')

element = driver.find_element('id', 'my-element-id')

elements = driver.find_elements('css selector', 'div.item')

button = driver.find_element('xpath', '//button[@type="submit"]')

button.click()

search_box = driver.find_element('name', 'q')
search_box.send_keys('Python Selenium')

BY Data Science Jupyter Notebooks