열심히 삽니다.

더 이상 귀찮아서 안하려고 했지만...

마소 사이트는 의외로 꽤 자주 열다가 뻑나기 때문에

사실 정확한 이유는 모르겠지만;;;

열때 테그못 찾는 거보니 잘못 열린거 같은데 정확한 이유야 마소 서버 관리자가 할일이고;;;

아무튼 사이트 열기에 실패할 때 예외 처리를 추가했습니다..

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

import smtplib, ssl
import email.utils
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

import sqlite3

from time import sleep

#import logging
#logging.basicConfig(level=logging.DEBUG, filename="test.log", format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')

options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
chrome_prefs = {}
options.experimental_options["prefs"] = chrome_prefs
chrome_prefs["profile.default_content_settings"] = {"images": 2}
chrome_prefs["profile.managed_default_content_settings"] = {"images": 2}
driver = webdriver.Chrome(executable_path="/usr/lib/chromium-browser/chromedriver", chrome_options=options)

dbPath = "./"
dbFileName = "PriceChk.sqlite3"
conn = sqlite3.connect(dbPath + dbFileName)

cur = conn.cursor()

SELECT_TEXT = "select B.ITEM_ID, B.NAME, A.SITE_TYPE, A.URL, A.URL_TYPE, B.URL_OPT1, B.URL_OPT2, B.URL_OPT3, "
SELECT_TEXT += "A.OBJ_TYPE_ONLYONE, A.OBJ_TYPE_ATTRIBUTE, A.OBJ_XPATH, A.OBJ_ATTRIBUTE, A.OBJ_TEXT_START, A.OBJ_TEXT_END "
SELECT_TEXT += "from SITE A, ITEM B where B.DEL=0 AND A.SITE_TYPE=B.SITE_TYPE order by A.URL"

dataList = cur.execute(SELECT_TEXT).fetchall()
for data in dataList:
    dataItemID = data[0]
    dataName = data[1]
    dataSiteType = data[2]
    dataUrl = data[3]
    dataUrlType = data[4]
    dataUrlOpt1 = data[5]
    dataUrlOpt2 = data[6] or ""
    dataUrlOpt3 = data[7] or ""

    dataObjTypeOnlyOne = data[8]
    dataObjTypeAttribute = data[9]
    dataObjXpath = data[10]
    dataObjAttribute = data[11]
    dataTextStart = data[12] or ""
    dataTextEnd = data[13] or ""

    url = ""
    if dataUrlType == 1:
        url = dataUrl.format(dataUrlOpt1)
    elif dataUrlType == 2:
        url = dataUrl.format(dataUrlOpt1, dataUrlOpt2)
    elif dataUrlType == 3:
        url = dataUrl.format(dataUrlOpt1, dataUrlOpt2, dataUrlOpt3)

    loopRun = True
    sleepTime = 0

    while loopRun:
        loopRun = False
        #print(url)
        if 50 < sleepTime:
            print("{0}:{1} Fail:{2}".format(dataItemID, dataName, url))
            break
        sleep(sleepTime)
        driver.get(url)

        targetObjs = driver.find_elements_by_xpath(dataObjXpath)
        if len(targetObjs) <= 0:
            loopRun = True
            sleepTime += 10
            print("{0}:{1} Fail Sleep Time:{2}".format(dataItemID, dataName, sleepTime))
            continue

        newPrice = ""
        if dataObjTypeOnlyOne == 1:
            targetObj = targetObjs[0]
            tempPrice = ""
            if dataObjTypeAttribute == 1:
                tempPrice = targetObj.get_attribute(dataObjAttribute)
            else:
                tempPrice = targetObj.text
            newPrice = tempPrice[len(dataTextStart):len(tempPrice)-len(dataTextEnd)]
        else:
            for targetObj in targetObjs:
                tempPrice = ""
                if dataObjTypeAttribute == 1:
                    tempPrice = targetObj.get_attribute(dataObjAttribute)
                else:
                    tempPrice = targetObj.text
                if tempPrice.startswith(dataTextStart) and tempPrice.endswith(dataTextEnd):
                    newPrice = tempPrice[len(dataTextStart):len(tempPrice)-len(dataTextEnd)]
                    break

        print("{0}:{1}:{2}".format(dataItemID, dataName, newPrice))

        oldPriceSelect = cur.execute("select ITEM_ID, PRICE, DATE from DATA where ITEM_ID = ? order by DATE desc limit 1", (str(dataItemID),)).fetchone()
        oldPrice = ""
        if not oldPriceSelect is None:
            oldPrice = oldPriceSelect[1]

        if oldPrice != newPrice:
            cur.execute("insert into DATA ( ITEM_ID, PRICE, DATE ) VALUES (?, ?, datetime('now', 'localtime'))", (dataItemID, newPrice))

            mailInfo = cur.execute("select SMTP_URL, SMTP_PORT, SEND_MAIL_ADDRESS, SEND_MAIL_PASSWORD, RECEIVE_MAIL_ADDRESS from MAILINFO").fetchone()
            if mailInfo != None:
                fromaddr = mailInfo[2]
                toaddr = mailInfo[4]

                msgBody = "{0}의 가격 변경\n{1}→{2}\n{3}".format(dataName, oldPrice, newPrice, url)
                msg = MIMEMultipart()
                msg['From'] = email.utils.formataddr(('가격봇', fromaddr))
                msg['To'] = email.utils.formataddr(('너님', toaddr))
                msg['Subject'] = "{0}의 가격 변경".format(dataName)
                msg.attach(MIMEText(msgBody))

                id = fromaddr
                password = mailInfo[3]

                context = ssl.create_default_context()
                with smtplib.SMTP(mailInfo[0], mailInfo[1]) as server:
                    server.ehlo() # Can be omitted
                    server.starttls(context=context)
                    server.ehlo() # Can be omitted
                    server.login(id, password)
                    server.sendmail(fromaddr, toaddr, msg.as_string())

conn.commit()
conn.close()
driver.close()
driver.quit()

더 이상 문제 생겨도 손 안될 생각임...

이거 고치고 여기 올릴지 고민하다가 그냥 올림...

쓸 사람도 없겠지만...

이상한 MYPI

Category

Profile

Counter