Commit @a7e40c447ddb604e1053dcf4236d857180ab7c13 - yjyoon/구미시-네이버-블로그-스크레퍼

윤영준 2023-12-11

updated cache clearing function

@a7e40c447ddb604e1053dcf4236d857180ab7c13

d46e704

a7e40c4

naver_blog_url_gatherer.py

--- naver_blog_url_gatherer.py

+++ naver_blog_url_gatherer.py

...	...	@@ -114,4 +114,6 @@
114	114	df_out = pd.DataFrame(df_data)
115	115	if not os.path.exists(f"blog/{keyword}"):
116	116	os.mkdir(f"blog/{keyword}")
117		- df_out.to_csv(f"blog/{keyword}/{dates}", index=False) (파일 끝에 줄바꿈 문자 없음)
	117	+
	118	+ if len(df_out) > 0 :
	119	+ df_out.to_csv(f"blog/{keyword}/{dates}", index=False) (파일 끝에 줄바꿈 문자 없음)

d46e704

a7e40c4

naver_cafe_info_gatherer.py

--- naver_cafe_info_gatherer.py

+++ naver_cafe_info_gatherer.py


 from webdriver_manager.chrome import ChromeDriverManager
 from selenium.webdriver.chrome.service import Service as ChromeService
 from tqdm import tqdm
+from joblib import Parallel, delayed
+from utils.cache_clear import delete_cache
 
 HEADER = {"User-Agent": "Mozilla/119.0 (Windows NT 10.0; Win64; x64) Chrome/98.0.4758.102"}
 

         ret["cafe_name"].append(cafe_name)
         ret["post_date"].append(post_date)
         ret["text"].append(text)
-
+    delete_cache(driver)
+    driver.close()
     return ret
 
 def merge_dicts(dict_list):

     return ranges
 
 
-if __name__ == "__main__":
-    from joblib import Parallel, delayed
-
-    keyword = "구미시"
+def naver_cafe_scrapper(keyword, step = 100, start_page_num=1, browser_thread_count=12):
     url = get_url(keyword, 1)
     total_page_num = get_total_page(url)
 
-    for scope in range_of_ranges(1, total_page_num+1, 100):
+    for scope in range_of_ranges(start_page_num, total_page_num+1, step):
         def parallel_scraping(keyword):
             urls = Parallel(n_jobs=-1)(delayed(get_url)(keyword, i) for i in range(scope[0], scope[1]))
-            results = Parallel(n_jobs=12)(delayed(scrap_agent)(url) for url in urls)
+            results = Parallel(n_jobs=browser_thread_count)(delayed(scrap_agent)(url) for url in urls)
             return results
 
         ret = parallel_scraping(keyword)

             os.mkdir(f"cafe/{keyword}")
         out_df.to_csv(f"cafe/{keyword}/{scope[0]}-{scope[1]}.csv", index=False)
         pass
-#    cafe_url_scrapper()
+
+if __name__ == "__main__":
+    naver_cafe_scrapper("구미시장")

a7e40c4

utils/cache_clear.py (added)

+++ utils/cache_clear.py

...	...	@@ -0,0 +1,27 @@
	1	+import undetected_chromedriver as uc
	2	+from selenium.webdriver.common.action_chains import ActionChains
	3	+from selenium.webdriver.common.keys import Keys
	4	+from webdriver_manager.chrome import ChromeDriverManager
	5	+from time import sleep
	6	+
	7	+
	8	+options = uc.ChromeOptions()
	9	+driver = uc.Chrome(driver_executable_path=ChromeDriverManager().install(), options=options)
	10	+
	11	+def delete_cache(driver):
	12	+ driver.execute_script("window.open('')") # Create a separate tab than the main one
	13	+ driver.switch_to.window(driver.window_handles[-1]) # Switch window to the second tab
	14	+ driver.get('chrome://settings/clearBrowserData') # Open your chrome settings.
	15	+ sleep(3)
	16	+ actions = ActionChains(driver)
	17	+ actions.key_down(Keys.SHIFT).send_keys(Keys.TAB * 6).key_up(Keys.SHIFT) #select "all time" browsing data
	18	+ actions.perform()
	19	+ sleep(3)
	20	+ actions.send_keys(Keys.DOWN * 5 + Keys.TAB * 7 + Keys.ENTER) #click on "clear data" button
	21	+ actions.perform()
	22	+ sleep(3)
	23	+ driver.close()
	24	+ driver.switch_to.window(driver.window_handles[0])
	25	+ print("Succesfully cleared the browsing data")
	26	+
	27	+# delete_cache(driver)(파일 끝에 줄바꿈 문자 없음)

Add a comment

Open 0
Closed 0

List

...	...	@@ -12,6 +12,8 @@
12	12	from webdriver_manager.chrome import ChromeDriverManager
13	13	from selenium.webdriver.chrome.service import Service as ChromeService
14	14	from tqdm import tqdm
	15	+from joblib import Parallel, delayed
	16	+from utils.cache_clear import delete_cache
15	17
16	18	HEADER = {"User-Agent": "Mozilla/119.0 (Windows NT 10.0; Win64; x64) Chrome/98.0.4758.102"}
17	19
...	...	@@ -103,7 +105,8 @@
103	105	ret["cafe_name"].append(cafe_name)
104	106	ret["post_date"].append(post_date)
105	107	ret["text"].append(text)
106		-
	108	+ delete_cache(driver)
	109	+ driver.close()
107	110	return ret
108	111
109	112	def merge_dicts(dict_list):
...	...	@@ -141,17 +144,14 @@
141	144	return ranges
142	145
143	146
144		-if __name__ == "__main__":
145		- from joblib import Parallel, delayed
146		-
147		- keyword = "구미시"
	147	+def naver_cafe_scrapper(keyword, step = 100, start_page_num=1, browser_thread_count=12):
148	148	url = get_url(keyword, 1)
149	149	total_page_num = get_total_page(url)
150	150
151		- for scope in range_of_ranges(1, total_page_num+1, 100):
	151	+ for scope in range_of_ranges(start_page_num, total_page_num+1, step):
152	152	def parallel_scraping(keyword):
153	153	urls = Parallel(n_jobs=-1)(delayed(get_url)(keyword, i) for i in range(scope[0], scope[1]))
154		- results = Parallel(n_jobs=12)(delayed(scrap_agent)(url) for url in urls)
	154	+ results = Parallel(n_jobs=browser_thread_count)(delayed(scrap_agent)(url) for url in urls)
155	155	return results
156	156
157	157	ret = parallel_scraping(keyword)
...	...	@@ -161,4 +161,6 @@
161	161	os.mkdir(f"cafe/{keyword}")
162	162	out_df.to_csv(f"cafe/{keyword}/{scope[0]}-{scope[1]}.csv", index=False)
163	163	pass
164		-# cafe_url_scrapper() (파일 끝에 줄바꿈 문자 없음)
	164	+
	165	+if __name__ == "__main__":
	166	+ naver_cafe_scrapper("구미시장") (파일 끝에 줄바꿈 문자 없음)

Delete comment