import csv
import json
import os
import pandas as pd
import requests
import sys
import argparse
import datetime
import urllib3
from selenium import webdriver
from urllib.parse import urlparse, urljoin
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.chrome.options import Options
# Initialize the Selenium webdriver
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
# Configure Chrome options to clear cache and disable extensions
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--disable-web-security')
chrome_options.add_argument('--disable-extensions')
chrome_options.add_argument('--allow-running-insecure-content')

# Set the download directory for PDFs
download_directory = "/home/a/zhoujl/Downloads/all_web_crawler_pdf"  # Replace with the actual directory
prefs = {"download.default_directory": download_directory}
chrome_options.add_experimental_option("prefs", prefs)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


successed_links = {}
visited_links = set()
broken_links = {}
denied_links = {}
broken_image = {}
redirect_to_home = {}
successed_image = set()
merged_broken_links = {}

max_depth = 0
input_url = ''
main_domain = ''
target_path = ''

class SimpleLink:
    href = ""
    css_class = ""
    link_text = ""
    def __init__(self, href, css_class, link_text):
        self.href = href
        self.css_class = css_class
        self.link_text = link_text
# Function to configure and retrieve WebDriver with authenticated session
def get_authenticated_driver(username, password, login_url):
    driver = webdriver.Chrome(options=chrome_options)

    # Navigate to the login page
    driver.get(login_url)
    if username != '' or password != '':
        # Fill in and submit the login form
        username_field = driver.find_element(By.NAME, 'user')
        password_field = driver.find_element(By.NAME, 'pass')
        login_button = driver.find_element(By.XPATH, '//input[@type="submit"]')

        username_field.send_keys(username)
        password_field.send_keys(password)
        login_button.click()

        # Wait for the login to complete, you might need to adjust this
        driver.implicitly_wait(2000)

    # Retrieve and return the cookies
    cookies = driver.get_cookies()
    return driver, cookies

def check_webpage(url, target_string, session, cookies=None):
    try:
        response = session.get(url, cookies=cookies, verify=False, timeout=15)
        response.raise_for_status()  # Raise an exception for bad status codes
        content = response.text

        if target_string in content:
            return True, content
        else:
            return False, content
    except requests.exceptions.RequestException as e:
        return False, str(e)

# Set the main domain
def is_internal_link(link):
    parsed_link = urlparse(link)

    return parsed_link.netloc == main_domain and parsed_link.path.startswith(target_path) or parsed_link.netloc == ""


def find_broken_links(driver, url, current_depth, cookies, session, requests_cookies, target_string):
    global successed_links
    global visited_links
    global broken_links
    global denied_links
    global broken_image
    global successed_image
    global redirect_to_home
    simpleLinkList = []


    if current_depth == 0:
        return "done"

    try:
        driver.get(url)
    except WebDriverException as e:
        print(f"Error accessing URL {url}: {e}")
    if url.endswith(".pdf") or url in broken_links or "type=1" in url or "download.php" in url:
        driver.back()
    else:
        link_elements = driver.find_elements(By.TAG_NAME, "a")
        images = [img.get_attribute("src") for img in driver.find_elements(By.TAG_NAME, "img")]
        for link_element in link_elements:
            simple_link = SimpleLink(link_element.get_attribute("href"), link_element.get_attribute("class"), link_element.text)
            simpleLinkList.append(simple_link)
        #links = [a for a in driver.find_elements(By.TAG_NAME, "a")]
        for simple_link in simpleLinkList:
            link = simple_link.href
            cssClass = simple_link.css_class
            link_text = simple_link.link_text
            if link == 'https://www.math.uzh.ch/compmath/en/konferenzdetails0?L=1&key1=755':

             pass
            try:#links = [a.get_attribute("href") for a in driver.find_elements(By.TAG_NAME, "a")]
                # link = link_element.get_attribute("href")
                # link_class = link_element.get_attribute("class")
            #for link in links:

                checkBool, content = check_webpage(link, target_string, session, cookies=requests_cookies)
                if checkBool and link not in visited_links:
                    visited_links.add(link)
                    response = requests.head(link, verify=False)
                    redirect_to_home.setdefault(link, [None, None, None, None, None])[0] = link
                    redirect_to_home[link][1] = response.status_code
                    redirect_to_home[link][2] = str(checkBool)
                    redirect_to_home[link][3] = url
                    redirect_to_home[link][4] = link_text
                if link and is_internal_link(link) and not link.startswith("mailto:"):
                    full_link = urljoin(url, link)
                    if full_link not in visited_links:
                            visited_links.add(full_link)
                            response = requests.head(full_link)
                            #checkBool, content = check_webpage(full_link, target_string, session, cookies=requests_cookies)
                            if response.status_code == 404:
                                broken_links.setdefault(full_link, [None, None, None, None, None])[0] = full_link
                                broken_links[full_link][1] = response.status_code
                                broken_links[full_link][2] = str(checkBool)
                                broken_links[full_link][3] = url
                                broken_links[full_link][4] = link_text
                               # print(f"Broken link: {full_link}")
                            if response.status_code == 403:
                                denied_links.setdefault(full_link, [None, None, None, None, None])[0] = full_link
                                denied_links[full_link][1] = response.status_code
                                denied_links[full_link][2] = str(checkBool)
                                denied_links[full_link][3] = url
                                denied_links[full_link][4] = link_text
                            else:
                                find_broken_links(driver, full_link, current_depth -1, cookies, session, requests_cookies, target_string)
                                if full_link not in broken_links:
                                    successed_links.setdefault(full_link, [None, None, None, None, None])[0] = full_link
                                    successed_links[full_link][1] = response.status_code
                                    successed_links[full_link][2] = str(checkBool)
                                    successed_links[full_link][3] = url
                                    # print(full_link)# Recursively check this link
                                    # print("keine fehler")
                                    # print('current_depth is:', current_depth)
                            #if checkBool == True:
                             #   print('Path redirects to home')
                            # print("---------------------------------------")
            except Exception as e:
                print(f"Error checking link {full_link}: {e}")

        for image in images:
            if image and is_internal_link(image):
                full_image_link = urljoin(url, image)
                if full_image_link not in visited_links:
                    visited_links.add(full_image_link)
                    try:
                        response = requests.head(full_image_link)
                        if response.status_code == 404:
                            broken_image.setdefault(full_image_link, [None, None, None, None, None])[0] = full_image_link
                            broken_image[full_image_link][1] = response.status_code
                            broken_image[full_image_link][2] = ''
                            broken_image[full_image_link][3] = url
                            broken_image[full_image_link][4] = "text"
                            #print(f"Broken image: {full_image_link}")
                        else:
                            successed_image.add(full_image_link)
                    except Exception as e:
                         print(f"Error checking image {full_image_link}: {e}")
    pass



def print_header(log_file, blacklists):
    current_datetime = datetime.datetime.now()
    if log_file.endswith("detail.log"):
         with open(log_file, "a+") as file:
            file.write(f"\nSkript : {__file__}")
            file.write("\nDetails Log " + current_datetime.strftime("%Y-%m-%d %H:%M"))
            file.write("\nInput URL:" + input_url)
            file.write("\nDOMAIN:" + main_domain)
            file.write("\n-------------------------------------\n")
            file.write("\nBlacklist : \n")
            for blacklist in blacklists:
                file.write(f"\"{blacklist}\"\n")
            file.write("-------------------------------------\n")
    elif log_file.endswith("summary.log"):
        with open(log_file, "a+") as file:
            file.write(f"\nSkript : {__file__}")
            file.write("\nSummary Log " + current_datetime.strftime("%Y-%m-%d %H:%M"))
            file.write("\nInput URL:" + input_url)
            file.write("\nDOMAIN:" + main_domain)
            file.write("\n-------------------------------------\n")
    else:
        return  print("No summary.log or detail.log are found")
    pass



def print_detail_log(links, title, detail_log_file):
    with open(detail_log_file, "a") as file:
        file.write("\n" + title + ":\n\n")
        if len(links) == 0:
            file.write("Nothing found.\n---------------------------------------\n")
        for full_link, details in links.items():
            target_link = details[0]
            response = details[1]  # assuming the status_code is at index 0
            redirect = details[2]  # assuming the redirect flag is at index 1
            url = details[3]
            if title != 'Successed links':
                link_text = details[4]
            file.write("Page URL: " + url + "\n")
            file.write("Target URL: " + target_link + "\n")
            file.write("RESPONSE: " + str(response) + "\n")
            if title != 'Successed links':
                file.write("Link Text: " + link_text + "\n")
            if title != 'Broken images' and redirect == True:
                file.write("REDIRECT_TO_HOME: " + str(redirect) + "\n")
            file.write("---------------------------------------\n")
    pass

def print_summary(redirect_to_home,checked_links,broken_links,checked_image,broken_image, summary_log_file, title):
    with open(summary_log_file, "a") as file:
        file.write(f"\n Redirect to Home: {len(redirect_to_home)}")
        file.write(f"\n Checked links: {len(checked_links)+len(broken_links)}")
        file.write(f"\n Broken links: {len(broken_links)}")
        file.write(f"\n Checked image: {len(checked_image)+len(broken_image)}")
        file.write(f"\n Broken image: {len(broken_image)}")
        file.write("\n---------------------------------------")
        if len(broken_links) > 0 or len(broken_image) > 0:
            file.write(f"\nSome issue in \"{title}\"")
        else:
            file.write("\nEverything is ok\n\n")
    pass

def write_broken_links_to_csv(broken_links, csv_file):
    header = ["Response Code", "PAGE URL", "Link Text", "Target URL"]

    with open(csv_file, "a", newline="", encoding="utf-8") as file:
        # Check if the file is empty

        is_empty = file.tell() == 0
        writer = csv.writer(file)

        if is_empty:
            writer.writerow(header)

        for link, details in broken_links.items():
            target_url = details[0]
            response_code = details[1]
            url = details[3]
            link_text = details[4]
            writer.writerow([response_code, url, link_text, target_url])

def clear_arrays():
    successed_links.clear()
    visited_links.clear()
    broken_links.clear()
    denied_links.clear()
    broken_image.clear()
    successed_image.clear()
    merged_broken_links.clear()
    redirect_to_home.clear()
    pass

def process_site(selected_site):
    global main_domain
    global target_path
    global visited_links
    global input_url

    max_depth = selected_site.get("max_depth")
    input_url = selected_site.get("startUrl")
    main_domain = urlparse(input_url).netloc
    target_path = selected_site.get("target_path")

    # Load the blacklist URLs from the JSON data
    blacklist = selected_site.get("blacklist", [])

    # Initialize the visited_links set with the URLs from the blacklist
    visited_links = set(blacklist)

    # Login credentials
    login_url = selected_site.get("login_url") or input_url
    username = selected_site.get("username")
    password = selected_site.get("password")
    target_string = selected_site.get("target_string")

    session = requests.Session()
    # target_string = 'Login/out'
    driver, cookies = get_authenticated_driver(username, password, login_url)
    requests_cookies = {cookie['name']: cookie['value'] for cookie in cookies}
    # current_datetime = datetime.datetime.now()
    # formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M")

    parsed_url = urlparse(input_url)
    detail_file_name = "log_files/" + parsed_url.netloc + parsed_url.path.replace("/", "_") + "_detail.log"
    detail_log_file = os.path.join(os.getcwd(), detail_file_name)

    summary_log_file = "log_files/" + parsed_url.netloc + parsed_url.path.replace("/", "_") + "_summary.log"
    title_summary = parsed_url.netloc + parsed_url.path

    # print header in files
    print_header(detail_log_file, blacklist)
    print_header(summary_log_file, blacklist)
    # function find_broken_links
    find_broken_links(driver, input_url, max_depth, cookies, session, requests_cookies, target_string)
    merged_broken_links = {**broken_links, **broken_image}

    # Clean up
    driver.quit()

    print_summary(redirect_to_home, successed_links, broken_links, successed_image, broken_image, summary_log_file,
                  title_summary)
    print_detail_log(redirect_to_home, 'Redirect to Home', detail_log_file)
    print_detail_log(broken_links, 'Broken links', detail_log_file)
    print_detail_log(broken_image, 'Broken images', detail_log_file)
    print_detail_log(denied_links, 'Denied links', detail_log_file)
    print_detail_log(successed_links, 'Successed links', detail_log_file)
    file_name = parsed_url.netloc + parsed_url.path.replace("/", "_")
    file_path = f"/home/a/zhoujl/Documents/broken_link_{file_name}.csv"
    write_broken_links_to_csv(merged_broken_links, file_path)
    print(f"Broken links found in {input_url}: {len(broken_links)}")
    clear_arrays()


def main():
    # if len(sys.argv) < 3 or not sys.argv[1].strip():
    #     print("Usage:python script.py conf.json <index or all>")
    #     sys.exit(1)
    #
    # parser = argparse.ArgumentParser(description='Web Crawling.')
    # parser.add_argument('conf_file', type=str, help='Path to the conf.json file')
    # #parser.add_argument('element_index', type=int, help='Index of the element to check in allSites')
    # args = parser.parse_args()

    conf_file_path = "conf.json"#args.conf_file

    with open(conf_file_path, "r") as conf_file:
        conf_data = json.load(conf_file)

    all_sites = conf_data.get("allSites", [])
    selected_arg = "9" #sys.argv[3]  # Get the third argument

    try:
        if selected_arg == "all":
            for selected_site in conf_data.get("allSites", []):
                process_site(selected_site)
        else:
            selected_index = int(selected_arg)
            if selected_index >= 0 and selected_index < len(conf_data.get("allSites", [])):
                selected_site = conf_data["allSites"][selected_index]
                process_site(selected_site)
            else:
                print("Invalid index.")
                sys.exit(1)
    except Exception as e:
       print(e)

if __name__ == "__main__":
    main()






