chrome_path = 'D:\pyCrawler\selenium_driver_chrome\chromedriver.exe'driver = webdriver.Chrome(chrome_path) driver.maximize_window() driver.set_page_load_timeout(60) driver.get(TigerUrl)
element = WebDriverWait(driver, 10, 0.5).until(EC.presence_of_element_located((By.ID,
'ControlGroupSearchView_ButtonSubmit')))
# 下拉選單需要拆段作業，先定位點擊，然後再巡覽選項# 出發機場el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchVieworiginStation1')) el.select_by_value('TPE') # 抵達機場el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchViewdestinationStation1')) el.select_by_value('DMK') # 去程日el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListMarketDay1')) el.select_by_value('21') # 去程年月el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListMarketMonth1')) el.select_by_value('2017-06') # 回程日el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListMarketDay2')) el.select_by_value('30') # 回程年月el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListMarketMonth2')) el.select_by_value('2017-06') # 成人數el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListPassengerType_ADT')) el.select_by_value('3') # 兒童數el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListPassengerType_CHD')) el.select_by_value('0') # 嬰兒數el = Select(driver.find_element_by_id('ControlGroupSearchView_AvailabilitySearchInputSearchView_DropDownListPassengerType_INFANT')) el.select_by_value('0') # 按下獲取航班driver.find_element_by_id('ControlGroupSearchView_ButtonSubmit').click() # 透過等待的設定來待網頁，看到需求的元件就往下執行了!WebDriverWait(driver, 10, 0.5).until(EC.presence_of_element_located((By.ID, 'flightSpinner')))
soup = BeautifulSoup(driver.page_source, 'html5lib') tbs = soup.select('.select-flight') start_lightPrice = soup.select('#tableMarket1 > tr > td')[1].select('label > span') print('sl:', start_lightPrice) start_comboPrice = soup.select('#tableMarket1 > tr > td')[2].select('label > span') print('sc:', start_comboPrice) end_lightPrice = soup.select('#tableMarket2 > tr > td')[1].select('label > span') print('el:', end_lightPrice) end_comboPrice = soup.select('#tableMarket2 > tr > td')[2].select('label > span') print('ec:', end_comboPrice)
chrome_path = 'D:\pyCrawler\selenium_driver_chrome\chromedriver.exe'
driver = webdriver.Chrome(chrome_path)
driver.maximize_window()
driver.set_page_load_timeout(60)
driver.get(TigerUrl)
from selenium.webdriver.support.ui import Select
el = Select(driver.find_element_by_id('id')) el.select_by_value('option')
driver.find_element_by_id('id').click()
for option in driver.find_element_by_id('id').click():
if option.text == '':
option.click()
resp = requests.session() resp1 = resp.post('https://booking.tigerairtw.com/Search.aspx', data=form_data) resp2 = resp.get('https://booking.tigerairtw.com/SelectFlights.aspx') soup = BeautifulSoup(resp2, 'html5lib')
python -m pip install <filename>
import json
import jieba
from collections import Counter
from wordcloud import WordCloud
import matplotlib.pyplot as plt
jieba.set_dictionart('dict.txt.big') # 設置繁中字典檔，跟專案放一起就可以指到了
if __name__ == '__main__':
justDoit()
def justDoit():
with open('abc.json', 'r', encoding='utf-8') as f:
data = json.load(f) # 案例假裝是從json檔取資料
dicts = list()
for d in data.values():
dicts += [s for s in jieba.cut(d) if s.split() and len(s) > 1]
# 當斷詞之後的文字非空格而且長度大於1
countD = Counter(dicts) #Counter是上面import的一個lib，很好用，會回傳list內的出現次數
print(countD) # 就可以列印出各詞出現的次數了
# 接著是文字雲，如果是處理中文字，就要附上字型檔，與專案置於同一資料夾即可
wCloud = WordCloud(font_path='中文字型檔').generate(' '.join(dicts))
plt.imshow(wCloud)
plt.axis('off')
plt.show()
items = []
with open('abc.csv', 'w', encoding='utf-8', newline='') as f:
writer = csv.writer(f) # 建立一個寫入物件 writer.writerow(('a', 'b', 'c')) # 標題欄 for item in items: writer.writerow((column for column in item))
with open('ezprice.csv', 'r', encoding='utf-8') as f:
reader = csv.DictReader(f) # 建立一個讀取物件
for row in reader:
print(row['a'], row['b'], row['c'])
for article in articles: page = get_articles(ptt_url + article['href']) if page: img_urls = parse(page) # 這邊主要要取得圖片的連結 saveImage(img_urls, article['title']) # 這邊要保存圖片
def parse(dom):
soup = BeautifulSoup(dom, 'html.parser')
links = soup.find(id='main-conetent').find_all('a')
imgurls = []
for link in links:
if re.match(r'^https?://(i.)?(m.)?imgur.com',link['href']):
imgurls.append(link['href'])
return imgurls
def saveImage(img_urls, title): if img_urls: try: dname = title.replace('?', '').replace('？', '').replace(' ', '')
.replace('Re:', '').strip() # 用 strip() 去除字串前後的空白 os.makedirs(dname) # 創建資料夾 for img_url in img_urls: # 主要是把下載連結的格式再調整 if img_url.split('//')[1].startswith('m.'): img_url = img_url.replace('//m.', '//i.') if not img_url.split('//')[1].startswith('i.'): img_url = img_url.split('//')[0] + '//i.' + img_url.split('//')[1] if not img_url.endswith('.jpg'): img_url += '.jpg' fname = img_url.split('/')[-1] urllib.request.urlretrieve(img_url, os.path.join(dname, fname)) except Exception as e: print(e)
country_to_count = dict()
for article in articles: print('文章IP:', article['title']) page = get_articles(ptt_url + article['href']) if page: ip = get_ip(page) country = get_country(ip) if country in country_to_count.keys(): country_to_count[country] += 1 else: country_to_count[country] = 1
def get_ip(SoureceIP):
reIP = '來自： \d+\.\d+.\d+.\d+' # 這是把頁面內的發文ip取出的正則式
match = re.search(reIP,SourceIP) # 確認文章內是否有相對應的格式
if match:
return match.group(0).replace('來自： ','') # 回傳第一個，另外也取代掉『來自： 』
else:
return None
def get_country(ip):
if(ip):
data = json.loads(request.get('http://freegeoip.net/json/' + ip).text)
countryName = data['country_name'] if data['country_name'] else None
return countryName
return None
resp = requests.get(
url=url, cookies={'over18': '1'}
)
import requests import time import json from bs4 import BeautifulSoup
ptt_url = 'https://www.ptt.cc' # 保留以後可以變數帶入其它板的機會
def __name__ == '__main__':
connect_page = checkStatus(ptt_url + '/bbs/sex/index.html') #確認網頁狀態是否正常if connect_page:
articles = [] #用來記錄文章list
today = time.strftime("%m/%d").lstrip('0') #今天日期格式調整，time是import的模組
today_articles,pre_url = get_articles(connect_page,today) #呼叫爬蟲fucntion
# 到上面，已經可以爬到主頁上的文章了!
# 爬完了之後，要進入驗證的迴圈，確認有無其它本日文章!
while today_articles: #確認有無本日文章，如果list回來已經沒有東西了，那就代表沒有了!
articles += today_articles #把爬回傳的list先寫丟進去主list
connect_page = checkStatus(pre_url) #一樣需要做網頁狀態驗證
today_articles,pre_url = get_articles(connect_page,today) #再呼叫爬蟲!
# 主要確認網頁是否正常
def checkStatus(url):
resp = requests.get(url = url,cookies = {'over18':'1'})if resp.status_code != 200: #如果網頁狀態不存活了，就直接return Nonereturn Noneelse:return resp.text
# 資料爬取的主要程式
def get_articles(respText,date): #一個是resp拋過來的資料，一個是今日的日期。
soup = BeautifulSoup(respText,'html5lib')
#取得網頁的上頁連結
pre_div = soup.find('div','btn-group btn-group-paging')
pre_url = pre_div.find_all('a')[1].href
articles = [] #用來記錄文章list
divs = soup.find_all('div','r-ent') #ptt的文章，都放在class是r-ent上!
for d in divs:
if d.find('div','date').text.strip() == date: #確認日期跟今天的日期是否相同
#取得推文數
pushCount = 0pushStr = d.find('div','nrec').text
#因為推文有時候會是『X』被噓爆，或是『爆』被推爆，所以要特別處理。if pushStr:try:pushCount = int(pushStr) #字串轉數字except ValueError:if pushStr =="爆":pushCount=99elif pushStr.startswith('X'):pushCount=-10#取得文章連結跟標題#這邊可以依需求自取，甚至可以再另外寫一個function去爬發文者的ip來做記錄!#後面可以利用那發文ip來比對id，就知道誰有什麼分身了...if d.find('a'): #有超連結就代表文章還活著，沒有被d掉。href = d.find('a').href #連結articleTitle = d.find('a').text #標題author = ''articles.append({'title':articleTitle,'href':href,'author':author,'pushCount':pushCount})return articles,pre_url