import random
import re
import regex
import time
import math
import pandas as pd
import os
import bs4
import warnings
from bs4 import MarkupResemblesLocatorWarning
from datetime import datetime, timedelta
from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service as ChromeService
from tqdm import tqdm
from joblib import Parallel, delayed
from utils.cache_clear import delete_cache


# To slience the MarkupResemblesLocatorWarning of bs4, which is annoying since I KNOW EVERYTHING IS HTML
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning, module='bs4')

HEADER = {"User-Agent": "Mozilla/119.0 (Windows NT 10.0; Win64; x64) Chrome/98.0.4758.102"}


def remove_html(strr):
    # print(strr)
    cleaning = regex.sub(r'<.*?>', '', strr.strip().replace("\n",' '))
    cleaning = regex.sub(r' +', ' ', cleaning)
    return cleaning

# Function to remove tags
def remove_tags(html):
    # parse html content
    soup = bs4.BeautifulSoup(html, "html.parser")

    for data in soup(['style', 'script']):
        # Remove tags
        data.decompose()

    # return data by retrieving the tag content
    return ' '.join(soup.stripped_strings)


def get_url(keyword, start_date, end_date, page_num, by_sim=True):
    if by_sim:
        orderBy = "sim"
    else:
        orderBy = "recentdate"
    url = f"https://section.blog.naver.com/Search/Post.naver?pageNo={page_num}&rangeType=PERIOD&orderBy={orderBy}&startDate={start_date}&endDate={end_date}&keyword={keyword}"
    return url


def get_article_count(plain_html):
    regex_match_count = regex.search(r'\d*(,\d{3}|d{3})*\b*건\b*<', plain_html)
    regex_match_count = regex.sub(r'건\b*<', '', regex_match_count.group())
    match_count = regex_match_count.replace(',', '')
    return int(match_count)


def get_total_page(url):
    # Chrome WebDriver 설치 경로 가져오기
    chrome_path = ChromeDriverManager().install()
    # WebDriver 설정
    options = webdriver.ChromeOptions()
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option("useAutomationExtension", False)
    options.add_argument('headless')
    options.add_argument('window-size=1920x1080')
    options.add_argument("disable-gpu")
    # options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")

    # WebDriver 초기화
    driver = webdriver.Chrome(service=ChromeService(chrome_path), options=options)
    driver.get(url)
    time.sleep(3)
    plain_html = driver.page_source
    article_count = get_article_count(plain_html)
    total_page = math.ceil(int(article_count) / 7)
    return total_page


def scrap_agent(url):
    # Chrome WebDriver 설치 경로 가져오기
    chrome_path = ChromeDriverManager().install()
    # WebDriver 설정
    options = webdriver.ChromeOptions()
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option("useAutomationExtension", False)
    options.add_argument('--blink-settings=imagesEnabled=false')
    options.add_argument('headless')
    options.add_argument('window-size=1920x1080')
    options.add_argument("disable-gpu")
    # options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")

    # WebDriver 초기화
    driver = webdriver.Chrome(service=ChromeService(chrome_path), options=options)
    driver.get(url)
    time.sleep(1+2*random.random())
    single_page_search_result = driver.find_elements(By.CSS_SELECTOR, ".area_list_search")

    search_result_plain_html = single_page_search_result[0].get_attribute('innerHTML')
    formatter = bs4.formatter.HTMLFormatter(indent=1)
    search_result_plain_html_beautified = bs4.BeautifulSoup(search_result_plain_html, 'html.parser').prettify(
        formatter=formatter)

    search_result_contents = regex.findall(
        r'(<!-- ngRepeat: post in postSearchCtrl.resultList -->)?'
        r'(.*?)'
        r'<!-- end ngRepeat: post in postSearchCtrl.resultList -->',
        search_result_plain_html_beautified,
        re.DOTALL
    )

    ret = {
        "url": [],
        "title": [],
        "blog_name": [],
        "post_date": [],
        "text": []
    }
    for content in search_result_contents:
        content = content[1].replace("\n", "")
        href = regex.search(
            r"https:\/\/blog\.naver\.com\/[\uAC00-\uD7AFa-zA-Z0-9_-]+\/[\uAC00-\uD7AFa-zA-Z0-9_-]+",
            content, re.DOTALL)
        # title is using bit different approach since the search engine highlights for search keywords
        title = regex.search(r'<strong class="title_post">(.*?)<\/span>', content)
        text = regex.search(r'<!-- ngIf: post.contents -->(.*?)<\/a>', content)
        author_name = regex.search(r'<em class="name_author">(.*?)<\/em>', content)
        post_date = regex.search(r'<span class="date">(.*?)<\/span>', content)
        href = href.group(0)
        title = remove_html(title.group(1))
        text = remove_tags(text.group(1))
        author_name = remove_tags(author_name.group(1))
        post_date = remove_tags((post_date.group(1)))

        ret["url"].append(href)
        ret["title"].append(title)
        ret["blog_name"].append(author_name)
        ret["post_date"].append(post_date)
        ret["text"].append(text)
    delete_cache(driver)
    driver.close()
    return ret


def merge_dicts(dict_list):
    # Initialize the result dictionary with empty lists
    result = {
        "url": [],
        "title": [],
        "blog_name": [],
        "post_date": [],
        "text": []
    }

    # Iterate through each dictionary and merge the lists
    for d in dict_list:
        for key in result.keys():
            result[key].extend(d.get(key, []))

    return result


def generate_date_range(start_date, end_date, interval):
    """
    :param start_date: start date in datetimestr formatted in %Y-%m-%d
    :param end_date: end date in datetimestr formatted in %Y-%m-%d
    :param interval: interval of time in DAYS
    :return: returns list of time interval that will be feeded into naver_blog_scrapper.
    for example, [[2023-10-01, 2023-10-07],[2023-10-08, 2023-10-14]] for generate_date_range("2023-10-01", "2023-10-14", 7)
    Also, this function can handle when the time range of start and end date is not perfectly divisible by interval
    generate_date_range("2023-10-01", "2023-10-14", 10) will produce output [['2023-10-01', '2023-10-10'], ['2023-10-11', '2023-10-14']]
    """

    # Convert the start and end date strings to datetime objects
    start = datetime.strptime(start_date, "%Y-%m-%d")
    end = datetime.strptime(end_date, "%Y-%m-%d")

    # Generate a range of dates from start to end
    date_ranges = []
    current_date = start
    while current_date < end:
        current_end = min(current_date + timedelta(days=interval - 1), end)
        date_ranges.append([current_date.strftime("%Y-%m-%d"), current_end.strftime("%Y-%m-%d")])
        current_date = current_end + timedelta(days=1)

    return date_ranges

def range_of_ranges(start, end, step):
    ranges = []
    current_start = start

    while current_start <= end:
        # Calculate the end of the current range
        current_end = min(current_start + step - 1, end)

        # Append the current range as a list
        ranges.append([current_start, current_end])

        # Update the start for the next range
        current_start += step

    return ranges


def naver_blog_scrapper(keyword, start_day, end_day, date_interval, page_step=10, start_page_num=1, browser_thread_count=12):
    url = get_url(keyword, start_day, end_day, 1)
    total_page_num = get_total_page(url)
    print(f"processing : {keyword}, during {start_day} to {end_day}")
    print(f"total_page_num : {total_page_num}")

    date_range = generate_date_range(start_day, end_day, date_interval)


    for i, date_range in enumerate(date_range):
        url = get_url(keyword, date_range[0], date_range[1], 1)
        total_page_num = get_total_page(url)
        task_pool = range_of_ranges(1, total_page_num+1, page_step)
        for task in task_pool:
            def parallel_scraping(keyword):
                urls = Parallel(n_jobs=-1)(delayed(get_url)(keyword, date_range[0], date_range[1], i) for i in range(task[0], task[1]))
                results = Parallel(n_jobs=browser_thread_count)(delayed(scrap_agent)(url) for url in urls)
                return results

            ret = parallel_scraping(keyword)
            merged_result = merge_dicts(ret)
            out_df = pd.DataFrame.from_dict(merged_result)
            if not os.path.exists(f"blog/{keyword}"):
                os.mkdir(f"blog/{keyword}")
            out_df.to_csv(f"blog/{keyword}/{date_range[0]}-{date_range[1]}%{task[0]}-{task[1]}.csv", index=False)




if __name__ == "__main__":
    #TODO start_page_num must be not working as intended
    naver_blog_scrapper("구미 송정동", "2023-02-05", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("구미 비산동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("도량동", "2023-02-05", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("공단동", "2022-07-21", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("광평동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("상모사곡동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("임오동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("인동동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("진미동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)
    naver_blog_scrapper("양포동", "2022-01-01", "2023-10-31", 100, 50, 1, 12)