爬虫学习1

最近学习了下爬虫，测试可以通过，先留着

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service

from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import urllib.request
import json
import os
import cv2

from webdriver_manager.chrome import ChromeDriverManager

def login_and_get_uid():

    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--disable-gpu')
    chrome_options.add_argument('--disable-dev-shm-usage')

    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)
    # try:
    #     pass
    # except Exception as e:
    #     print("eee:", e)
    # finally:
    #     driver.quit()
    driver.get("https://yyy.xxxxx.com/login")
    print(driver.title)
    driver.implicitly_wait(3)
    print(driver.title)
    #         print("User Agent:", user_agent)
    driver.find_element(By.NAME, 'username').send_keys("yyyyy")
    driver.find_element(By.NAME, 'password').send_keys("xxxx")

    # 找到登录按钮并点击
    login_button = driver.find_element(By.CLASS_NAME, 'btn-submit')
    login_button.click()

    time.sleep(1)

    img_element = driver.find_element(By.XPATH, '//img[@class="validate_big"]')
    src_value = img_element.get_attribute("src")
    print(src_value)
    img_element_2 = driver.find_element(By.XPATH, '//img[@class="validate_block"]')

    src_value_2 = img_element_2.get_attribute("src")
    print(src_value_2)
    if os.path.exists('./src_value.png'):
        os.remove('./src_value.png')
    urllib.request.urlretrieve(src_value, './src_value.png')
    if os.path.exists('./src_value_2.png'):
        os.remove('./src_value_2.png')
    urllib.request.urlretrieve(src_value_2, './src_value_2.png')

    smallImage = driver.find_element(By.XPATH, '//*[@id="slide"]/div/div[2]/div[2]')

    newDis = template_matching('./src_value.png', './src_value_2.png')

    print(newDis)
    driver.implicitly_wait(5)
    ActionChains(driver).click_and_hold(smallImage).perform()
    ActionChains(driver).move_by_offset(xoffset=newDis, yoffset=0).perform()
    ActionChains(driver).release().perform()
    time.sleep(2)
    print("ab:")
    time.sleep(2)
    driver.get("https:/www.sohuc.om")
    time.sleep(3)
    # xAuth = driver.execute_script("return window.sessionStorage.getItem('x-auth');")
    # access = driver.execute_script("return window.sessionStorage.getItem('access');")
    #
    max_ane = {
        "token": driver.execute_script("return window.sessionStorage.getItem('x-auth');"),
        "user_agent": driver.execute_script("return navigator.userAgent;"),
        "site_info": json.loads(driver.execute_script("return window.sessionStorage.getItem('access');")),
    }

    return {"max_ane": max_ane, "fin_ane": None}

def template_matching(img_path, tm_path):
    # 导入图片，灰度化
    img_rgb = cv2.imread(img_path)
    template_rgb = cv2.imread(tm_path)
    img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)
    tm_gray = cv2.cvtColor(template_rgb, cv2.COLOR_BGR2GRAY)
    # 缺口图去除背景
    h, w = tm_gray.shape
    w_start_index, h_start_index = 0, 0
    w_end_index, h_end_index = w, h
    # 缺口图去除背景
    # 算出高起始位置
    for i in range(h):
        if not any(tm_gray[i, :]):
            h_start_index = i
        else:
            break
    # 算出高的结束位置
    for i in range(h - 1, 0, -1):
        if not any(tm_gray[i, :]):
            h_end_index = i
        else:
            break
    # 算出宽的起始位置
    for i in range(w):
        if not any(tm_gray[:, i]):
            w_start_index = i
        else:
            break
    # 算出宽的起始位置
    for i in range(w - 1, 0, -1):
        if not any(tm_gray[:, i]):
            w_end_index = i
        else:
            break
    # 取出完整的缺口图
    tm_gray = tm_gray[h_start_index:h_end_index + 1, w_start_index:w_end_index + 1]
    # 自适应阈值话
    img_thresh = cv2.adaptiveThreshold(img_gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 5, 0)
    tm_thresh = cv2.adaptiveThreshold(tm_gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 5, 0)

    # 边缘检测
    img_canny = cv2.Canny(img_thresh, 0, 500)
    tm_canny = cv2.Canny(tm_thresh, 0, 500)
    #     cv2.imshow("img_canny", img_canny)
    #     cv2.imshow("tm_canny", tm_canny)
    h, w = tm_gray.shape[:2]
    # 模板匹配
    res = cv2.matchTemplate(img_canny, tm_canny, cv2.TM_CCOEFF_NORMED)

    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
    left_up = max_loc
    print(left_up)
    right_bottom = (max_loc[0] + w, max_loc[1] + h)  # 右下角
    # 圈出矩形坐标
    cv2.rectangle(img_rgb, max_loc, right_bottom, (0, 0, 255), 2)
    if os.path.exists('res.png'):
        os.remove('res.png')
    cv2.imwrite('res.png', img_rgb)

    # 显示图片 参数：（窗口标识字符串，imread读入的图像）
    # cv2.imshow("test_image", img_rgb)
    return max_loc[0] / 400 * 238

if __name__ == '__main__':
    re = login_and_get_uid()
    print("re : ", re)