Commit @3ac64686feb4e872ab6bfc69beaaf1b8eafaea87 - yjyoon/구미시-네이버-블로그-스크레퍼

윤영준 2023-12-13

Existing cafe info scrapper does not work due to Naver's lazyness, as only first 100 pages are relevent and rest of it is just repeated. This is an rework of that.

@3ac64686feb4e872ab6bfc69beaaf1b8eafaea87

aaf31da

3ac6468

DEPRECIATED_naver_cafe_info_gatherer.py (Renamed from naver_cafe_info_gatherer.py)

--- naver_cafe_info_gatherer.py

+++ DEPRECIATED_naver_cafe_info_gatherer.py


 HEADER = {"User-Agent": "Mozilla/119.0 (Windows NT 10.0; Win64; x64) Chrome/98.0.4758.102"}
 
 
+
+
 # Function to remove tags
 def remove_tags(html):
     # parse html content

aaf31da

3ac6468

cafe_scrapping_executor.py (Renamed from cafe_scarapping_executor.py)

--- cafe_scarapping_executor.py

+++ cafe_scrapping_executor.py

...	...	@@ -1,7 +1,7 @@
1		-from naver_blog_url_gatherer import blog_url_scrapper
	1	+from DEPRECIATED_naver_cafe_info_gatherer import naver_cafe_scrapper
2	2
3	3	if __name__ == "__main__":
4		- # blog_url_scrapper("선산읍", "2022-01-01", "2023-10-31")
	4	+ naver_cafe_scrapper("선산읍", 100, 101)
5	5	# blog_url_scrapper("고아읍", "2022-01-01", "2023-10-31")
6	6	# blog_url_scrapper("산동읍", "2022-01-01", "2023-10-31")
7	7	# blog_url_scrapper("무을면", "2022-01-01", "2023-10-31")
...	...	@@ -25,4 +25,4 @@
25	25	# blog_url_scrapper("인동동", "2022-01-01", "2023-10-31")
26	26	# blog_url_scrapper("진미동", "2022-01-01", "2023-10-31")
27	27	# blog_url_scrapper("양포동", "2022-01-01", "2023-10-31")
28		- blog_url_scrapper("구미", "2022-01-01", "2023-10-31") (파일 끝에 줄바꿈 문자 없음)
	28	+ # blog_url_scrapper("구미", "2022-01-01", "2023-10-31") (파일 끝에 줄바꿈 문자 없음)

aaf31da

3ac6468

naver_blog_info_gatherer.py

--- naver_blog_info_gatherer.py

+++ naver_blog_info_gatherer.py

...	...	@@ -1,3 +1,4 @@
	1	+import random
1	2	import re
2	3	import regex
3	4	import time
...	...	@@ -96,7 +97,7 @@
96	97	# WebDriver 초기화
97	98	driver = webdriver.Chrome(service=ChromeService(chrome_path), options=options)
98	99	driver.get(url)
99		- time.sleep(2)
	100	+ time.sleep(1+2*random.random())
100	101	single_page_search_result = driver.find_elements(By.CSS_SELECTOR, ".area_list_search")
101	102
102	103	search_result_plain_html = single_page_search_result[0].get_attribute('innerHTML')
...	...	@@ -236,4 +237,13 @@
236	237
237	238	if __name__ == "__main__":
238	239	#TODO start_page_num must be not working as intended
239		- naver_blog_scrapper("도개면", "2022-10-28", "2023-10-31", 100, 50, 1, 12) (파일 끝에 줄바꿈 문자 없음)
	240	+ naver_blog_scrapper("구미 송정동", "2023-02-05", "2023-10-31", 100, 50, 1, 12)
	241	+ naver_blog_scrapper("구미 비산동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
	242	+ naver_blog_scrapper("도량동", "2023-02-05", "2023-10-31", 100, 50, 1, 12)
	243	+ naver_blog_scrapper("공단동", "2022-07-21", "2023-10-31", 100, 50, 1, 12)
	244	+ naver_blog_scrapper("광평동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
	245	+ naver_blog_scrapper("상모사곡동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
	246	+ naver_blog_scrapper("임오동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
	247	+ naver_blog_scrapper("인동동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
	248	+ naver_blog_scrapper("진미동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
	249	+ naver_blog_scrapper("양포동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)

3ac6468

naver_cafe_inifinityscroll_from_searchengine.py (added)

+++ naver_cafe_inifinityscroll_from_searchengine.py

...	...	@@ -0,0 +1,211 @@
	1	+import random
	2	+import re
	3	+import regex
	4	+import time
	5	+import math
	6	+import pandas as pd
	7	+import os
	8	+import bs4
	9	+from bs4 import MarkupResemblesLocatorWarning
	10	+from datetime import datetime, timedelta
	11	+from selenium import webdriver
	12	+from selenium.webdriver.common.by import By
	13	+from webdriver_manager.chrome import ChromeDriverManager
	14	+from selenium.webdriver.chrome.service import Service as ChromeService
	15	+from tqdm import tqdm
	16	+from joblib import Parallel, delayed
	17	+from utils.cache_clear import delete_cache
	18	+import warnings
	19	+
	20	+
	21	+def get_url(keyword, start_date, end_date):
	22	+
	23	+ url = (f"https://search.naver.com/search.naver?where=articleg&query={keyword}"
	24	+ f"&ie=utf8&st=rel&date_option=8"
	25	+ f"&date_from={start_date.replace('-','.')}"
	26	+ f"&date_to={end_date.replace('-','.')}"
	27	+ f"&srchby=text&dup_remove=1"
	28	+ f"&cafe_url=&without_cafe_url="
	29	+ f"&sm=tab_opt&nso=so%3Add%2Cp%3Afrom{start_date.replace('-','')}to{end_date.replace('-','')}&nso_open=1&prdtype=0")
	30	+ return url
	31	+
	32	+def remove_tags(html):
	33	+ # parse html content
	34	+ soup = bs4.BeautifulSoup(html, "html.parser")
	35	+
	36	+ for data in soup(['style', 'script']):
	37	+ # Remove tags
	38	+ data.decompose()
	39	+
	40	+ # return data by retrieving the tag content
	41	+ return ' '.join(soup.stripped_strings)
	42	+
	43	+def infinity_scroll_to_the_bottom(driver):
	44	+ # Automatically scroll the page
	45	+ MAXITER = 2000
	46	+ scroll_pause_time = 0.1 + random.random() * 2 # Pause between each scroll
	47	+ screen_height = driver.execute_script("return window.screen.height;") # Browser window height
	48	+ i = 1
	49	+ while i < MAXITER:
	50	+ # Scroll down
	51	+ driver.execute_script(f"window.scrollTo(0, {screen_height * i + random.randint(1, 500)});")
	52	+ i += 1
	53	+ time.sleep(scroll_pause_time)
	54	+
	55	+ # Check if reaching the end of the page
	56	+ scroll_height = driver.execute_script("return document.body.scrollHeight;")
	57	+ # print(scroll_height)
	58	+ if screen_height * i > scroll_height:
	59	+ break
	60	+ return driver
	61	+
	62	+def scrap_agent(url, keyword, start_date, end_date):
	63	+ # Chrome WebDriver 설치 경로 가져오기
	64	+ chrome_path = ChromeDriverManager().install()
	65	+ # WebDriver 설정
	66	+ options = webdriver.ChromeOptions()
	67	+ options.add_experimental_option("excludeSwitches", ["enable-automation"])
	68	+ options.add_experimental_option("useAutomationExtension", False)
	69	+ options.add_argument('headless')
	70	+ options.add_argument('window-size=1920x1080')
	71	+ options.add_argument("disable-gpu")
	72	+ # options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
	73	+
	74	+ # WebDriver 초기화
	75	+ driver = webdriver.Chrome(service=ChromeService(chrome_path), options=options)
	76	+ driver.get(url)
	77	+ time.sleep(2)
	78	+
	79	+ infinity_scroll_to_the_bottom(driver)
	80	+
	81	+ single_page_search_result = driver.find_elements(By.CSS_SELECTOR, ".lst_view")
	82	+
	83	+ search_result_plain_html = single_page_search_result[0].get_attribute('innerHTML')
	84	+ formatter = bs4.formatter.HTMLFormatter(indent=1)
	85	+ search_result_plain_html_beautified = bs4.BeautifulSoup(search_result_plain_html, 'html.parser').prettify(formatter=formatter)
	86	+
	87	+ search_result_contents = regex.findall(r'<li class="bx">(.*?)\n<\/li>', search_result_plain_html_beautified, re.DOTALL)
	88	+
	89	+ ret = {
	90	+ "url": [],
	91	+ "title": [],
	92	+ "cafe_name": [],
	93	+ "post_date": [],
	94	+ "text": []
	95	+ }
	96	+ for content in search_result_contents:
	97	+ content = content.replace("\n", "")
	98	+ href = "/".join(str(i) for i in regex.findall(r'href="([^"]+)/([^"]+)/([^"]+)/([^"]+)"', content, re.DOTALL)[0])
	99	+ title = regex.search(r'<div class="title_area">(.*?) </div>',content)
	100	+ text = regex.search(r' <div class="dsc_area">(.*?) <\/div>',content)
	101	+ cafe_name = regex.search(r' <div class="user_info">(.*?) </div>',content)
	102	+ post_date = regex.search(r'<span class="sub">(.*?)<\/span>',content)
	103	+ title = remove_tags(title.group(1))
	104	+ title = title[:-11] # to remove date
	105	+ text = remove_tags(text.group(1))
	106	+ cafe_name = remove_tags(cafe_name.group(1))
	107	+ post_date = remove_tags((post_date.group(1)))
	108	+
	109	+ ret["url"].append(href)
	110	+ ret["title"].append(title)
	111	+ ret["cafe_name"].append(cafe_name)
	112	+ ret["post_date"].append(post_date)
	113	+ ret["text"].append(text)
	114	+ delete_cache(driver)
	115	+ driver.close()
	116	+
	117	+ # merged_result = merge_dicts(ret)
	118	+ out_df = pd.DataFrame.from_dict(ret)
	119	+ if not os.path.exists(f"cafe/{keyword}"):
	120	+ os.mkdir(f"cafe/{keyword}")
	121	+ out_df.to_csv(f"cafe/{keyword}/{start_date}-{end_date}.csv", index=False)
	122	+
	123	+ return ret
	124	+
	125	+def merge_dicts(dict_list):
	126	+ # Initialize the result dictionary with empty lists
	127	+ result = {
	128	+ "url": [],
	129	+ "title": [],
	130	+ "blog_name": [],
	131	+ "post_date": [],
	132	+ "text": []
	133	+ }
	134	+
	135	+ # Iterate through each dictionary and merge the lists
	136	+ for d in dict_list:
	137	+ for key in result.keys():
	138	+ result[key].extend(d.get(key, []))
	139	+
	140	+ return result
	141	+
	142	+
	143	+def generate_date_range(start_date, end_date, interval):
	144	+ """
	145	+ :param start_date: start date in datetimestr formatted in %Y-%m-%d
	146	+ :param end_date: end date in datetimestr formatted in %Y-%m-%d
	147	+ :param interval: interval of time in DAYS
	148	+ :return: returns list of time interval that will be feeded into naver_blog_scrapper.
	149	+ for example, [[2023-10-01, 2023-10-07],[2023-10-08, 2023-10-14]] for generate_date_range("2023-10-01", "2023-10-14", 7)
	150	+ Also, this function can handle when the time range of start and end date is not perfectly divisible by interval
	151	+ generate_date_range("2023-10-01", "2023-10-14", 10) will produce output [['2023-10-01', '2023-10-10'], ['2023-10-11', '2023-10-14']]
	152	+ """
	153	+
	154	+ # Convert the start and end date strings to datetime objects
	155	+ start = datetime.strptime(start_date, "%Y-%m-%d")
	156	+ end = datetime.strptime(end_date, "%Y-%m-%d")
	157	+
	158	+ # Generate a range of dates from start to end
	159	+ date_ranges = []
	160	+ current_date = start
	161	+ while current_date < end:
	162	+ current_end = min(current_date + timedelta(days=interval - 1), end)
	163	+ date_ranges.append([current_date.strftime("%Y-%m-%d"), current_end.strftime("%Y-%m-%d")])
	164	+ current_date = current_end + timedelta(days=1)
	165	+
	166	+ return date_ranges
	167	+
	168	+
	169	+def naver_cafe_scrapper(keyword, start_date, end_date, interval, browser_thread_count=1):
	170	+ date_ranges = generate_date_range(start_date, end_date, interval)
	171	+ urls = []
	172	+ for date_range in date_ranges:
	173	+ urls.append(get_url(keyword, date_range[0], date_range[1]))
	174	+ pass
	175	+ def parallel_scraping(keyword):
	176	+ results = Parallel(n_jobs=browser_thread_count)(delayed(scrap_agent)(url, keyword, date_ranges[i][0], date_ranges[i][1]) for i, url in enumerate(urls))
	177	+ return results
	178	+
	179	+ ret = parallel_scraping(keyword)
	180	+
	181	+
	182	+if __name__ == "__main__":
	183	+ naver_cafe_scrapper("선산읍", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	184	+ naver_cafe_scrapper("고아읍", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	185	+ naver_cafe_scrapper("산동읍", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	186	+ naver_cafe_scrapper("도개면", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	187	+ naver_cafe_scrapper("장천면", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	188	+ naver_cafe_scrapper("무을면", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	189	+ naver_cafe_scrapper("해평면", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	190	+ naver_cafe_scrapper("옥성면", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	191	+ naver_cafe_scrapper("구미 송정동", start_date="2022-01-01", end_date="2023-10-31", interval=10, browser_thread_count=1)
	192	+ naver_cafe_scrapper("원평동", start_date="2022-01-01", end_date="2023-10-31", interval=10, browser_thread_count=1)
	193	+ naver_cafe_scrapper("구미 지산동", start_date="2022-01-01", end_date="2023-10-31", interval=10, browser_thread_count=1)
	194	+ naver_cafe_scrapper("도량동", start_date="2022-01-01", end_date="2023-10-31", interval=10, browser_thread_count=1)
	195	+ naver_cafe_scrapper("선주원남동", start_date="2022-01-01", end_date="2023-10-31", interval=10, browser_thread_count=1)
	196	+ naver_cafe_scrapper("신평1동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	197	+ naver_cafe_scrapper("신평2동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	198	+ naver_cafe_scrapper("형곡1동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	199	+ naver_cafe_scrapper("형곡2동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	200	+ naver_cafe_scrapper("구미 비산동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	201	+ naver_cafe_scrapper("공단동", start_date="2022-01-01", end_date="2023-10-31", interval=10, browser_thread_count=1)
	202	+ naver_cafe_scrapper("광평동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	203	+ naver_cafe_scrapper("상모사곡동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	204	+ naver_cafe_scrapper("임오동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	205	+ naver_cafe_scrapper("인동동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	206	+ naver_cafe_scrapper("진미동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	207	+ naver_cafe_scrapper("양표동", start_date="2022-01-01", end_date="2023-10-31", interval=30, browser_thread_count=1)
	208	+
	209	+ naver_cafe_scrapper("구미", start_date="2022-01-01", end_date="2023-10-31", interval=1, browser_thread_count=1)
	210	+ naver_cafe_scrapper("구미시장", start_date="2022-01-01", end_date="2023-10-31", interval=7, browser_thread_count=1)
	211	+

Add a comment

Open 0
Closed 0

List

...	...	@@ -24,6 +24,8 @@
24	24	HEADER = {"User-Agent": "Mozilla/119.0 (Windows NT 10.0; Win64; x64) Chrome/98.0.4758.102"}
25	25
26	26
	27	+
	28	+
27	29	# Function to remove tags
28	30	def remove_tags(html):
29	31	# parse html content

Delete comment