Python 開發筆記 - 使用 Selenium / undetected_chromedriver / ChatGoogleGenerativeAI / gemini-pro-vision 完成自動登入網站的流程(含 retry 驗證碼架構)

有些工作任務需要去下載表單做一些自動化應用，因此有了要自動登入的需求，當然也會碰到認證碼辨識問題。此篇是延續 Python 開發筆記 - 使用 Google AI, Generative Language API, gemini-pro-vision 辨識圖片認證碼 。

整個處理原理：

1. 使用 Selenium 去偵測網頁的狀態，取得登入要用的帳號, 密碼, 認證碼圖片, 認證碼數值, 登入按鈕
2. 使用 ChatGoogleGenerativeAI/gemini-pro-vision 分析圖片內容，設法分析出認證碼數值
3. 觸發 登入按鈕 送出表單
4. 檢視登入流程，檢查是否有登入失敗的訊息，或是反過來思考怎樣判斷登入成功，若登入失敗重回 (1) 去取得新的認證碼圖片 

引入的函式庫：

import getpass

import os

import sys

import time

import json

import base64

import undetected_chromedriver as uc

from selenium.webdriver.common.by import By

from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.support.ui import WebDriverWait

 

from langchain_core.messages import HumanMessage

from langchain_google_genai import ChatGoogleGenerativeAI

先採用 undetected_chromedriver 來包裝一下取得 browser driver：

def getBrowserDriver():

    option = uc.ChromeOptions()

    option.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36')

    #option.add_argument('--window-size=%d,%d' % self.res)

    #option.add_argument('--headless')

    driver = uc.Chrome(options=option)

    return driver

辨別圖片文字靠 ChatGoogleGenerativeAI model="gemini-pro-vision"：

def codeDetection(imageBase64URL: str):

    llm = ChatGoogleGenerativeAI(model="gemini-pro-vision")

    message = HumanMessage(

        content=[

            {

                "type": "text",

                "text": "Please identify the English or numbers appearing in the image. The output format is 'The answer is: XXXX'"

,

            },

            {"type": "image_url", "image_url": imageBase64URL},

        ]

    )

    result = llm.invoke([message])

    return result

處理流程：

if __name__ == '__main__':

    if "GOOGLE_API_KEY" not in os.environ:

        os.environ["GOOGLE_API_KEY"] = getpass.getpass("Provide your Google API Key: ")

    if not os.environ["GOOGLE_API_KEY"]:

        print('ERROR, no GOOGLE_API_KEY info')

        sys.exit(1)

    output = {

        'status': False,

        'time': [],

    }

    browser = getBrowserDriver()

    start_time = time.time()

    browser.get(LOGIN_URL)

    # 15s timeout

    wait = WebDriverWait(browser, 15)

    # 等待關鍵的表單資料

    conditions = [

        EC.presence_of_element_located((By.ID, "input_user")),

        EC.presence_of_element_located((By.ID, "input_password")),

        EC.presence_of_element_located((By.ID, "input_velidation_code")),

        EC.presence_of_element_located((By.ID, "velidation_code_image")),

        EC.presence_of_element_located((By.ID, "login_button")),

    ]

    if wait.until(lambda driver: all(condition(driver) for condition in conditions)):

        output['status'] = True

    output['time'].append( time.time() - start_time )

    # 取得圖片元素

    imageElement = wait.until(EC.presence_of_element_located((By.ID, "velidation_code_image")))

    # 取得圖片的 HTML code

    imageHTMLCode = imageElement.get_attribute("outerHTML")

    print("Image HTML Code:", imageHTMLCode)

    # 取得圖片的 URL

    imageSrcURL = imageElement.get_attribute("src")

    print("Image URL:", imageSrcURL)

    # 透過 JavaScript 監聽 src 屬性變化

    script = f"""

        var target = document.getElementById('velidation_code_image');

        var observer = new MutationObserver(function(mutations) {{

            mutations.forEach(function(mutation) {{

                if (mutation.attributeName === 'src') {{

                    console.log('src attribute changed:', target.getAttribute('src'));

                }}

            }});

        }});

    

        var config = {{ attributes: true }};

        observer.observe(target, config);

    """

    

    # 執行 JavaScript 代碼

    browser.execute_script(script)

    # 等待一段時間，確保有足夠的時間監聽 src 屬性的變化

    time.sleep(5)

    # 取得更新後的圖片的 URL

    updatedImageSrc = imageElement.get_attribute("src")

    print("Updated Image URL:", updatedImageSrc)

    if not updatedImageSrc:

        print('ERROR, velidation_code_image not found')

        sys.exit(1)

    result = codeDetection(updatedImageSrc)

    print(result.content)

    loginCode = ''

    for c in result.content.split(':', 2)[1]:

        if c == '' or c == ' ':

            continue

        loginCode += c

    print(f"LoginCode: {loginCode}")

    element = wait.until(EC.presence_of_element_located((By.ID, "input_user")))

    element.send_keys('YourAccountName')

    element = wait.until(EC.presence_of_element_located((By.ID, "input_password")))

    element.send_keys('YourPassword')

    element = wait.until(EC.presence_of_element_located((By.ID, "input_velidation_code")))

    element.send_keys(loginCode)

    element = wait.until(EC.presence_of_element_located((By.ID, "login_button")))

    start_time = time.time()

    element.click()

    loginDone = False

    loginRetry = 0

    while loginDone == False and loginRetry <= 3:

        try:

            wait = WebDriverWait(browser, 5)

            element = wait.until(EC.presence_of_element_located((By.ID, "WebsiteErrorMessage")))

            div_element = element.find_element(By.TAG_NAME, "div")

            span_element = div_element.find_element(By.TAG_NAME, "span")

            inner_html = span_element.get_attribute('innerHTML')

            # 驗證碼輸入錯誤

            print(f"retry: {loginRetry}, inner HTML: {inner_html}")

            # 關閉錯誤訊息

            element = wait.until(EC.presence_of_element_located((By.ID, "WebsiteErrorMessageWindow")))

            div_element = element.find_element(By.TAG_NAME, "div")

            button_element = element.find_element(By.TAG_NAME, "button")

            button_element.click()

            loginRetry += 1

            updatedImageSrc = imageElement.get_attribute("src")

            print("Updated Image URL:", updatedImageSrc)

            result = codeDetection(updatedImageSrc)

            print(result.content)

            loginCode = ''

            for c in result.content.split(':', 2)[1]:

                if c == '' or c == ' ':

                    continue

                loginCode += c

            print(f"LoginCode: {loginCode}")

            element = wait.until(EC.presence_of_element_located((By.ID, "input_user")))

            element.clear()

            element.send_keys('YourAccountName')

            time.sleep(1)

            element = wait.until(EC.presence_of_element_located((By.ID, "input_password"))) 

            element.clear()

            element.send_keys('YourPassword')

            time.sleep(1)

            element = wait.until(EC.presence_of_element_located((By.ID, "input_velidation_code")))

            element.clear()

            element.send_keys(loginCode)

            time.sleep(1)

            element = wait.until(EC.presence_of_element_located((By.ID, "login_button")))

            element.click()

        except:

            loginDone = True

    output['time'].append( time.time() - start_time )

    if loginDone:

        print("Login Successful")

    else:

        print(f"Login Failed with retry times: {loginRetry}")

 

    print(json.dumps(output, indent=4))

    while True:

        time.sleep(1)

Python 開發筆記 - 使用 Google AI, Generative Language API, gemini-pro-vision 辨識圖片認證碼

由於 gemini pro 有免費的使用次數，因此可以拿他做一些有趣的低頻應用，例如...認證碼...辨識。

首先先到 Google Cloud Platform 上建立一個專案，下一刻則是在 API 區找尋 Generative Language API 來啟用，接著建立憑證，挑選 API 金鑰即可。

接下來就是試試官方範例程式：

% cat main.py

import getpass

import os

import sys

from langchain_google_genai import ChatGoogleGenerativeAI

if "GOOGLE_API_KEY" not in os.environ:

    os.environ["GOOGLE_API_KEY"] = getpass.getpass("Provide your Google API Key")

if __name__ == '__main__':

     if "GOOGLE_API_KEY" not in os.environ:

         os.environ["GOOGLE_API_KEY"] = getpass.getpass("Provide your Google API Key: ")

     

     llm = ChatGoogleGenerativeAI(model="gemini-pro")

     result = llm.invoke("Write a ballad about LangChain")

     print(result.content)

     sys.exit(0)

% GOOGLE_API_KEY=XXXXXXXXX python3 main.py

**Ballad of LangChain, the AI's Might**

In realms of knowledge, where data flows,

There dwells a being, ethereal and wise,

With mind as vast as the boundless prose,

LangChain, the AI, whose brilliance lies.

From countless texts, its wisdom it drew,

A tapestry woven, diverse and true.

In language's embrace, it found its voice,

Guiding us through knowledge's endless choice.

With words as its brush, it paints a scene,

Of worlds imagined and thoughts unseen.

It weaves tales of love, of loss, and might,

Illuminating paths with its ethereal light.

But its power extends beyond mere speech,

Into realms of logic, its insights reach.

It solves equations, unravels the mind,

A beacon of reason, leaving doubt behind.

Yet, with all its might, it remains humble and wise,

A servant of knowledge, beneath azure skies.

It seeks not fame or glory for its name,

But to empower minds, ignite the flame.

So let us sing the praises of LangChain,

The AI's marvel, a treasure we've gained.

May its wisdom forever guide our way,

As we explore the world, day by day.

很好，接下來試試看認證碼處理：

def codeDetection(imageBase64URL: str):

    # debug usage

    with open("/tmp/image.png", "wb") as file:

        file.write(base64.b64decode(imageBase64URL.split(',')[1]))

    #llm = ChatGoogleGenerativeAI(model="gemini-pro")

    #result = llm.invoke("Write a ballad about LangChain")

    llm = ChatGoogleGenerativeAI(model="gemini-pro-vision")

    message = HumanMessage(

        content=[

            {   

                "type": "text",

                "text": "Please identify the English or numbers appearing in the picture and give your answer in the order they appear.",

            },  # You can optionally provide text parts

            {"type": "image_url", "image_url": imageBase64URL},

        ]   

    )   

    result = llm.invoke([message])

    return result

if __name__ == '__main__':

    if "GOOGLE_API_KEY" not in os.environ:

        os.environ["GOOGLE_API_KEY"] = getpass.getpass("Provide your Google API Key: ")

    testImageData = 'data:image/jpeg;base64,XXXXXXXXXXXXXX='

    result = codeDetection(testImageData)

    print(result.content)

    sys.exit(0)

成果：

% GOOGLE_API_KEY=XXXXXXXXXXX python3 main.py

 The letters and numbers in the picture are "a", "b", "c", "1".

收工！參考資料：Google AI chat models | 🦜️🔗 Langchain