Module `lupro.config`

lurpo 全局配置

Expand source code

'''
lurpo 全局配置
'''

from .useragent import ua_list
import random
from datetime import datetime
import requests
import httpx
import os
import sys


__all__ = [
    'RUNFILE',
    'HTTP_ENGINE',
    'PERSISTENCE_ENABLED',
    'PERSISTENCE_PATH',
    'PROXIES',
    'VERIFY_PROXIES',
    'VERIFY_PROXIES',
    'get_header',
    'get_proxies',
    'logging',
    'verify_proxies'
]

# runfile
RUNFILE = os.path.split(sys.argv[0])[0]

# HTTP引擎
HTTP_ENGINE = httpx

# 对象持久化
PERSISTENCE_ENABLED = False

# 对象持久化存储路径
PERSISTENCE_PATH = 'endurance.db'

# 代理池
PROXIES = []

# 是否验证代理池
VERIFY_PROXIES = False

# 自定义请求头
def get_header(ualist = ua_list) -> str:
    '''`get_header` 生成一个UA头.

    Args:
        None : 无参数

    Returns:
        str : 返回一个UA头字符串即可
    '''
    return random.choice(ualist)

# 自定义代理池
def get_proxies():
    '''获取代理,并将代理赋值到 `lupro.config.PROXIES`.

    Args:
        None : 无参数

    Returns:
        bool : 返回Ture
    '''
    raise NameError('Please rewrite `get_proxies` function.')

# 自定义日志打印
def logging(text) -> str:
    '''自定义日志打印  `logging` 可以重写,返回带 `text` 的字符串即可.

    Args:
        `text` : str 日志
    
    Returns:
        str : 返回处理后的打印日志
    '''
    return "%s INFO %s"  % (datetime.now(), text)

# 自定义代理验证
def verify_proxies(proxies : str) -> str:
    ''' 自定义代理验证 `verify_proxies` 可以重写,如果可用返回 `proxies` 否则返回 `False`.

    Args:
        `proxies` : str 代理
    
    Returns:
        str | bool : 如果可用返回 `proxies` 否则返回 `False`
    '''

    try:
        if HTTP_ENGINE.__name__ == 'requests':
            HTTP_ENGINE.get(url='http://www.baidu.com/', headers={"User-Agent": get_header()}, proxies = {'http': f"//{proxies}"}, timeout=10)
        elif HTTP_ENGINE.__name__ == 'httpx':
            HTTP_ENGINE.get(url='http://www.baidu.com/', headers={"User-Agent": get_header()}, proxies = {'http://': f"http://{proxies}"}, timeout=10)
    except:
        return False
    return proxies

# 载入配置
try:
    import lupro_config
    for i in __all__:
        if hasattr(lupro_config, i):
            locals()[i] = getattr(lupro_config, i)
except ModuleNotFoundError:
    pass

Functions

def get_header(ualist=['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60', 'Opera/8.0 (Windows NT 5.1; U; en)', 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0', 'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 ', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 ', '(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1', 'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1', 'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24']) ‑> str

get_header() 生成一个UA头.

Args

None : 无参数

Returns

str: 返回一个UA头字符串即可

Expand source code

def get_header(ualist = ua_list) -> str:
    '''`get_header` 生成一个UA头.

    Args:
        None : 无参数

    Returns:
        str : 返回一个UA头字符串即可
    '''
    return random.choice(ualist)

def get_proxies()

获取代理,并将代理赋值到 lupro.config.PROXIES.

Args

None : 无参数

Returns

bool: 返回Ture

Expand source code

def get_proxies():
    '''获取代理,并将代理赋值到 `lupro.config.PROXIES`.

    Args:
        None : 无参数

    Returns:
        bool : 返回Ture
    '''
    raise NameError('Please rewrite `get_proxies` function.')

def logging(text) ‑> str

自定义日志打印 logging() 可以重写,返回带 text 的字符串即可.

Args

text : str 日志

Returns

str: 返回处理后的打印日志

Expand source code

def logging(text) -> str:
    '''自定义日志打印  `logging` 可以重写,返回带 `text` 的字符串即可.

    Args:
        `text` : str 日志
    
    Returns:
        str : 返回处理后的打印日志
    '''
    return "%s INFO %s"  % (datetime.now(), text)

def verify_proxies(proxies: str) ‑> str

自定义代理验证 verify_proxies() 可以重写,如果可用返回 proxies 否则返回 False.

Args

proxies : str 代理

Returns

str | bool : 如果可用返回 proxies 否则返回 False

Expand source code

def verify_proxies(proxies : str) -> str:
    ''' 自定义代理验证 `verify_proxies` 可以重写,如果可用返回 `proxies` 否则返回 `False`.

    Args:
        `proxies` : str 代理
    
    Returns:
        str | bool : 如果可用返回 `proxies` 否则返回 `False`
    '''

    try:
        if HTTP_ENGINE.__name__ == 'requests':
            HTTP_ENGINE.get(url='http://www.baidu.com/', headers={"User-Agent": get_header()}, proxies = {'http': f"//{proxies}"}, timeout=10)
        elif HTTP_ENGINE.__name__ == 'httpx':
            HTTP_ENGINE.get(url='http://www.baidu.com/', headers={"User-Agent": get_header()}, proxies = {'http://': f"http://{proxies}"}, timeout=10)
    except:
        return False
    return proxies