Commit @02d77bfbce98ffbf59773bf8977d39bdc78f6c91 - yjyoon/구미시-네이버-블로그-스크레퍼

윤영준 2023-12-11

updated cafe scrapper

@02d77bfbce98ffbf59773bf8977d39bdc78f6c91

36b794c

02d77bf

main.py

--- main.py

+++ main.py

...	...	@@ -1,28 +1,28 @@
1	1	from naver_blog_url_gatherer import blog_url_scrapper
2	2
3	3	if __name__ == "__main__":
4		- blog_url_scrapper("선산읍", "2022-01-01", "2023-10-31")
5		- blog_url_scrapper("고아읍", "2022-01-01", "2023-10-31")
6		- blog_url_scrapper("산동읍", "2022-01-01", "2023-10-31")
7		- blog_url_scrapper("무을면", "2022-01-01", "2023-10-31")
8		- blog_url_scrapper("옥성면", "2022-01-01", "2023-10-31")
9		- blog_url_scrapper("도개면", "2022-01-01", "2023-10-31")
10		- blog_url_scrapper("해평면", "2022-01-01", "2023-10-31")
11		- blog_url_scrapper("장천면", "2022-01-01", "2023-10-31")
12		- blog_url_scrapper("송정동", "2022-01-01", "2023-10-31")
13		- blog_url_scrapper("원평동", "2022-01-01", "2023-10-31")
14		- blog_url_scrapper("지산동", "2022-01-01", "2023-10-31")
15		- blog_url_scrapper("도량동", "2022-01-01", "2023-10-31")
16		- blog_url_scrapper("선주원남동", "2022-01-01", "2023-10-31")
17		- blog_url_scrapper("형곡1동", "2022-01-01", "2023-10-31")
18		- blog_url_scrapper("형곡2동", "2022-01-01", "2023-10-31")
19		- blog_url_scrapper("신평1동", "2022-01-01", "2023-10-31")
20		- blog_url_scrapper("신평2동", "2022-01-01", "2023-10-31")
21		- blog_url_scrapper("공단동", "2022-01-01", "2023-10-31")
22		- blog_url_scrapper("광평동", "2022-01-01", "2023-10-31")
23		- blog_url_scrapper("상모사곡동", "2022-01-01", "2023-10-31")
24		- blog_url_scrapper("임오동", "2022-01-01", "2023-10-31")
25		- blog_url_scrapper("인동동", "2022-01-01", "2023-10-31")
26		- blog_url_scrapper("진미동", "2022-01-01", "2023-10-31")
27		- blog_url_scrapper("양포동", "2022-01-01", "2023-10-31")
	4	+ # blog_url_scrapper("선산읍", "2022-01-01", "2023-10-31")
	5	+ # blog_url_scrapper("고아읍", "2022-01-01", "2023-10-31")
	6	+ # blog_url_scrapper("산동읍", "2022-01-01", "2023-10-31")
	7	+ # blog_url_scrapper("무을면", "2022-01-01", "2023-10-31")
	8	+ # blog_url_scrapper("옥성면", "2022-01-01", "2023-10-31")
	9	+ # blog_url_scrapper("도개면", "2022-01-01", "2023-10-31")
	10	+ # blog_url_scrapper("해평면", "2022-01-01", "2023-10-31")
	11	+ # blog_url_scrapper("장천면", "2022-01-01", "2023-10-31")
	12	+ # blog_url_scrapper("송정동", "2022-01-01", "2023-10-31")
	13	+ # blog_url_scrapper("원평동", "2022-01-01", "2023-10-31")
	14	+ # blog_url_scrapper("지산동", "2022-01-01", "2023-10-31")
	15	+ # blog_url_scrapper("도량동", "2022-01-01", "2023-10-31")
	16	+ # blog_url_scrapper("선주원남동", "2022-01-01", "2023-10-31")
	17	+ # blog_url_scrapper("형곡1동", "2022-01-01", "2023-10-31")
	18	+ # blog_url_scrapper("형곡2동", "2022-01-01", "2023-10-31")
	19	+ # blog_url_scrapper("신평1동", "2022-01-01", "2023-10-31")
	20	+ # blog_url_scrapper("신평2동", "2022-01-01", "2023-10-31")
	21	+ # blog_url_scrapper("공단동", "2022-01-01", "2023-10-31")
	22	+ # blog_url_scrapper("광평동", "2022-01-01", "2023-10-31")
	23	+ # blog_url_scrapper("상모사곡동", "2022-01-01", "2023-10-31")
	24	+ # blog_url_scrapper("임오동", "2022-01-01", "2023-10-31")
	25	+ # blog_url_scrapper("인동동", "2022-01-01", "2023-10-31")
	26	+ # blog_url_scrapper("진미동", "2022-01-01", "2023-10-31")
	27	+ # blog_url_scrapper("양포동", "2022-01-01", "2023-10-31")
28	28	blog_url_scrapper("구미", "2022-01-01", "2023-10-31") (파일 끝에 줄바꿈 문자 없음)

36b794c

02d77bf

naver_blog_url_gatherer.py

--- naver_blog_url_gatherer.py

+++ naver_blog_url_gatherer.py


     search_result_plain_html = single_page_search_result[0].get_attribute('innerHTML')
     search_result_urls = regex.findall(r"https:\/\/blog\.naver\.com\/[\uAC00-\uD7AFa-zA-Z0-9_]+\/[\uAC00-\uD7AFa-zA-Z0-9_]+",
                   search_result_plain_html)
+    # convenient single liner that removes duplicates
     search_result_urls = list(set(search_result_urls))
 
     return search_result_urls
 
 def blog_url_scrapper(keyword, start_date, end_date):
-
     dates_to_search = generate_date_range(start_date=start_date,end_date=end_date)
     iter_over_dates = tqdm(dates_to_search, position=0, leave=True)
     for dates in iter_over_dates:

02d77bf

naver_cafe_info_gatherer.py (added)

+++ naver_cafe_info_gatherer.py

...	...	@@ -0,0 +1,164 @@
	1	+import random
	2	+import re
	3	+import regex
	4	+import time
	5	+import math
	6	+import pandas as pd
	7	+import os
	8	+import bs4
	9	+from datetime import datetime, timedelta
	10	+from selenium import webdriver
	11	+from selenium.webdriver.common.by import By
	12	+from webdriver_manager.chrome import ChromeDriverManager
	13	+from selenium.webdriver.chrome.service import Service as ChromeService
	14	+from tqdm import tqdm
	15	+
	16	+HEADER = {"User-Agent": "Mozilla/119.0 (Windows NT 10.0; Win64; x64) Chrome/98.0.4758.102"}
	17	+
	18	+
	19	+# Function to remove tags
	20	+def remove_tags(html):
	21	+ # parse html content
	22	+ soup = bs4.BeautifulSoup(html, "html.parser")
	23	+
	24	+ for data in soup(['style', 'script']):
	25	+ # Remove tags
	26	+ data.decompose()
	27	+
	28	+ # return data by retrieving the tag content
	29	+ return ' '.join(soup.stripped_strings)
	30	+
	31	+def get_url(keyword, page_num):
	32	+ t_rand = 1702255000000 + random.randint(0,999999)
	33	+ url = f"https://section.cafe.naver.com/ca-fe/home/search/articles?q={keyword}&t={t_rand}&em=1&p={page_num}"
	34	+ return url
	35	+
	36	+def get_total_page(url):
	37	+ # Chrome WebDriver 설치 경로 가져오기
	38	+ chrome_path = ChromeDriverManager().install()
	39	+ # WebDriver 설정
	40	+ options = webdriver.ChromeOptions()
	41	+ options.add_experimental_option("excludeSwitches", ["enable-automation"])
	42	+ options.add_experimental_option("useAutomationExtension", False)
	43	+ options.add_argument('headless')
	44	+ options.add_argument('window-size=1920x1080')
	45	+ options.add_argument("disable-gpu")
	46	+ # options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
	47	+
	48	+ # WebDriver 초기화
	49	+ driver = webdriver.Chrome(service=ChromeService(chrome_path), options=options)
	50	+ driver.get(url)
	51	+ time.sleep(3)
	52	+ total_count = driver.find_element(By.CSS_SELECTOR, ".sub_text")
	53	+ total_count = total_count.text
	54	+ total_page = math.ceil(int(total_count.replace(',',''))/12)
	55	+ return total_page
	56	+
	57	+def scrap_agent(url):
	58	+ # Chrome WebDriver 설치 경로 가져오기
	59	+ chrome_path = ChromeDriverManager().install()
	60	+ # WebDriver 설정
	61	+ options = webdriver.ChromeOptions()
	62	+ options.add_experimental_option("excludeSwitches", ["enable-automation"])
	63	+ options.add_experimental_option("useAutomationExtension", False)
	64	+ options.add_argument('headless')
	65	+ options.add_argument('window-size=1920x1080')
	66	+ options.add_argument("disable-gpu")
	67	+ # options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
	68	+
	69	+ # WebDriver 초기화
	70	+ driver = webdriver.Chrome(service=ChromeService(chrome_path), options=options)
	71	+ driver.get(url)
	72	+ time.sleep(2)
	73	+ single_page_search_result = driver.find_elements(By.CSS_SELECTOR, ".item_list")
	74	+
	75	+ search_result_plain_html = single_page_search_result[0].get_attribute('innerHTML')
	76	+ formatter = bs4.formatter.HTMLFormatter(indent=1)
	77	+ search_result_plain_html_beautified = bs4.BeautifulSoup(search_result_plain_html, 'html.parser').prettify(formatter=formatter)
	78	+
	79	+ search_result_contents = regex.findall(r' <div class="article_item_wrap">(.*?)\n <\/div>', search_result_plain_html_beautified, re.DOTALL)
	80	+
	81	+ ret = {
	82	+ "url": [],
	83	+ "title": [],
	84	+ "cafe_name": [],
	85	+ "post_date": [],
	86	+ "text": []
	87	+ }
	88	+ for content in search_result_contents:
	89	+ content = content.replace("\n", "")
	90	+ href = regex.search(r"https:\/\/cafe\.naver\.com\/[\uAC00-\uD7AFa-zA-Z0-9_]+\?iframe_url\=[\uAC00-\uD7AFa-zA-Z0-9\.\%_]+", content, re.DOTALL)
	91	+ title = regex.search(r'<strong class="title">(.*?)<\/strong>',content)
	92	+ text = regex.search(r'<p class="text">(.*?)<\/p>',content)
	93	+ cafe_name = regex.search(r'<span class="cafe_name">(.*?)<\/span>',content)
	94	+ post_date = regex.search(r'<span class="date">(.*?)<\/span>',content)
	95	+ href = remove_tags(href.group(0))
	96	+ title = remove_tags(title.group(1))
	97	+ text = remove_tags(text.group(1))
	98	+ cafe_name = remove_tags(cafe_name.group(1))
	99	+ post_date = remove_tags((post_date.group(1)))
	100	+
	101	+ ret["url"].append(href)
	102	+ ret["title"].append(title)
	103	+ ret["cafe_name"].append(cafe_name)
	104	+ ret["post_date"].append(post_date)
	105	+ ret["text"].append(text)
	106	+
	107	+ return ret
	108	+
	109	+def merge_dicts(dict_list):
	110	+ # Initialize the result dictionary with empty lists
	111	+ result = {
	112	+ "url": [],
	113	+ "title": [],
	114	+ "cafe_name": [],
	115	+ "post_date": [],
	116	+ "text": []
	117	+ }
	118	+
	119	+ # Iterate through each dictionary and merge the lists
	120	+ for d in dict_list:
	121	+ for key in result.keys():
	122	+ result[key].extend(d.get(key, []))
	123	+
	124	+ return result
	125	+
	126	+
	127	+def range_of_ranges(start, end, step):
	128	+ ranges = []
	129	+ current_start = start
	130	+
	131	+ while current_start <= end:
	132	+ # Calculate the end of the current range
	133	+ current_end = min(current_start + step - 1, end)
	134	+
	135	+ # Append the current range as a list
	136	+ ranges.append([current_start, current_end])
	137	+
	138	+ # Update the start for the next range
	139	+ current_start += step
	140	+
	141	+ return ranges
	142	+
	143	+
	144	+if __name__ == "__main__":
	145	+ from joblib import Parallel, delayed
	146	+
	147	+ keyword = "구미시"
	148	+ url = get_url(keyword, 1)
	149	+ total_page_num = get_total_page(url)
	150	+
	151	+ for scope in range_of_ranges(1, total_page_num+1, 100):
	152	+ def parallel_scraping(keyword):
	153	+ urls = Parallel(n_jobs=-1)(delayed(get_url)(keyword, i) for i in range(scope[0], scope[1]))
	154	+ results = Parallel(n_jobs=12)(delayed(scrap_agent)(url) for url in urls)
	155	+ return results
	156	+
	157	+ ret = parallel_scraping(keyword)
	158	+ merged_result = merge_dicts(ret)
	159	+ out_df = pd.DataFrame.from_dict(merged_result)
	160	+ if not os.path.exists(f"cafe/{keyword}"):
	161	+ os.mkdir(f"cafe/{keyword}")
	162	+ out_df.to_csv(f"cafe/{keyword}/{scope[0]}-{scope[1]}.csv", index=False)
	163	+ pass
	164	+# cafe_url_scrapper()(파일 끝에 줄바꿈 문자 없음)

Add a comment

Open 0
Closed 0

List

...	...	@@ -81,12 +81,12 @@
81	81	search_result_plain_html = single_page_search_result[0].get_attribute('innerHTML')
82	82	search_result_urls = regex.findall(r"https:\/\/blog\.naver\.com\/[\uAC00-\uD7AFa-zA-Z0-9_]+\/[\uAC00-\uD7AFa-zA-Z0-9_]+",
83	83	search_result_plain_html)
	84	+ # convenient single liner that removes duplicates
84	85	search_result_urls = list(set(search_result_urls))
85	86
86	87	return search_result_urls
87	88
88	89	def blog_url_scrapper(keyword, start_date, end_date):
89		-
90	90	dates_to_search = generate_date_range(start_date=start_date,end_date=end_date)
91	91	iter_over_dates = tqdm(dates_to_search, position=0, leave=True)
92	92	for dates in iter_over_dates:

Delete comment