[Python] selenium

selenium

웹페이지 테스트 자동화용 모듈이다.

개발/테스트용 드라이버(웹브라우저)를 사용하여 실제 사용자가 사용하는 것처럼 동작한다.

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

from bs4 import BeautifulSoup
import time

 

 

1. python.org로 이동하여 자동으로 검색

chrome_driver = 'chromedriver 경로'
driver = webdriver.Chrome(chrome_driver)

driver.get('https://www.python.org/')

search = driver.find_element_by_id('id-search-field')
time.sleep(2)

search.clear()
time.sleep(2)

search.send_keys("lambda")
time.sleep(2)

search.send_keys(Keys.RETURN)
time.sleep(2)

driver.close()

 

 

2. 다음뉴스 웹사이트 크롤링

chrome_driver = 'chromedriver 경로'
driver = webdriver.Chrome(chrome_driver)

url = 'https://news.v.daum.net/v/20210821112100211'
driver.get(url)

src = driver.page_source

soup = BeautifulSoup(src)
comment = soup.select_one('span.alex-count-area')

driver.close()

comment.get_text()

 

 

3. 특정 element의 로딩 대기

chrome_driver = 'chromedriver 경로'
driver = webdriver.Chrome(chrome_driver)

url = 'https://news.naver.com/main/read.naver?mode=LSD&mid=shm&sid1=100&oid=005&aid=0001465638'
driver.get(url)

WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'em.simplecmt_num')))

src = driver.page_source

soup = BeautifulSoup(src)
comment = soup.select_one('em.simplecmt_num')

driver.close()

comment.get_text()