Commit @e754d841a82d16e626358f2d132d6b8f4f741824 - yjyoon/구미시-네이버-블로그-스크레퍼

윤영준 2023-12-12

added a line that prevents images from loading

@e754d841a82d16e626358f2d132d6b8f4f741824

32fe611

e754d84

naver_blog_info_gatherer.py

--- naver_blog_info_gatherer.py

+++ naver_blog_info_gatherer.py


     options = webdriver.ChromeOptions()
     options.add_experimental_option("excludeSwitches", ["enable-automation"])
     options.add_experimental_option("useAutomationExtension", False)
+    options.add_argument('--blink-settings=imagesEnabled=false')
     options.add_argument('headless')
     options.add_argument('window-size=1920x1080')
     options.add_argument("disable-gpu")

 def naver_blog_scrapper(keyword, start_day, end_day, date_interval, page_step=10, start_page_num=1, browser_thread_count=12):
     url = get_url(keyword, start_day, end_day, 1)
     total_page_num = get_total_page(url)
+    print(f"processing : {keyword}, during {start_day} to {end_day}")
     print(f"total_page_num : {total_page_num}")
 
     date_range = generate_date_range(start_day, end_day, date_interval)

 
 
 if __name__ == "__main__":
-    naver_blog_scrapper("선산읍", "2022-01-01", "2023-10-31", 7, 50, 1, 12)
+    naver_blog_scrapper("구미 송정동", "2022-01-01", "2023-10-31", 7, 50, 1, 12)

Add a comment

Open 0
Closed 0

List

...	...	@@ -81,6 +81,7 @@
81	81	options = webdriver.ChromeOptions()
82	82	options.add_experimental_option("excludeSwitches", ["enable-automation"])
83	83	options.add_experimental_option("useAutomationExtension", False)
	84	+ options.add_argument('--blink-settings=imagesEnabled=false')
84	85	options.add_argument('headless')
85	86	options.add_argument('window-size=1920x1080')
86	87	options.add_argument("disable-gpu")
...	...	@@ -200,6 +201,7 @@
200	201	def naver_blog_scrapper(keyword, start_day, end_day, date_interval, page_step=10, start_page_num=1, browser_thread_count=12):
201	202	url = get_url(keyword, start_day, end_day, 1)
202	203	total_page_num = get_total_page(url)
	204	+ print(f"processing : {keyword}, during {start_day} to {end_day}")
203	205	print(f"total_page_num : {total_page_num}")
204	206
205	207	date_range = generate_date_range(start_day, end_day, date_interval)
...	...	@@ -226,4 +228,4 @@
226	228
227	229
228	230	if __name__ == "__main__":
229		- naver_blog_scrapper("선산읍", "2022-01-01", "2023-10-31", 7, 50, 1, 12) (파일 끝에 줄바꿈 문자 없음)
	231	+ naver_blog_scrapper("구미 송정동", "2022-01-01", "2023-10-31", 7, 50, 1, 12) (파일 끝에 줄바꿈 문자 없음)

Delete comment