taiqiuxuanzhi_spider/setting.py

import time

# 爬虫的日志级别
LOG_LEVEL = 'INFO'

# 日志文件夹
LOG_DIR = 'logs'

# 爬虫的日志文件路径
LOG_FILE = '58spider.log'

# 日志格式
LOG_FORMAT = '%(asctime)s - %(filename)s - %(lineno)d - %(name)s - %(levelname)s - %(message)s'

# 爬虫的并发请求数量
CONCURRENT_REQUESTS = 3

# 爬虫的User-Agent
USER_AGENT = [
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
    # "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    # "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
    # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"
]

# 爬虫的IP代理
DOWNLOADER_IP_PROXY = 'https://api.xiaoxiangdaili.com/ip/get?appKey=1114815955776983040&appSecret=M4RS3NpN&cnt=2&wt=json&method=http&city=&province='

# 区名
_QU = 'jimo'
# 面积
_AREA = '100_300'
# 房租
_MONEY = '0_10000'
# 爬取的起始URL
START_URL = f'https://qd.58.com/{_QU}/shangpucz/pn1/?area={_AREA}&huansuanyue={_MONEY}'

# 持久化方式
STORE_METHOD = 'csv'

# csv文件夹
CSV_DIR = 'output_csv'

# csv文件名
CSV_FILENAME = f'{_QU}_{_AREA}_{_MONEY}_{int(time.time() * 1000)}.csv'

# 高德web服务API-key
GAODE_KEY = '515a64d5324a70ba9c5a95f9539370ec'

# 量化-半径
BANJING = '750'

# 量化-住宅个数
ZHUZHAI_COUNT = 3

# 量化-学校个数
XUEXIAO_COUNT = 2

# 量化-写字楼个数
XIEZILOU_COUNT = 1

# 量化-竞品门店个数
JINGPIN_COUNT = 2

# 量化-连锁品牌个数
LIANSUO_COUNT = 2

# 量化-连锁品牌名
LIANSUO_LIST = ['好想来', '萨么', '丹香', '赵一鸣', '零食很忙', '糖巢', '大家乐', '元祖', '米兰西饼', '85度C', '幸福西饼', '好利来']