| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869 |
- import time
- # 爬虫的日志级别
- LOG_LEVEL = 'INFO'
- # 日志文件夹
- LOG_DIR = 'logs'
- # 爬虫的日志文件路径
- LOG_FILE = '58spider.log'
- # 日志格式
- LOG_FORMAT = '%(asctime)s - %(filename)s - %(lineno)d - %(name)s - %(levelname)s - %(message)s'
- # 爬虫的并发请求数量
- CONCURRENT_REQUESTS = 3
- # 爬虫的User-Agent
- USER_AGENT = [
- "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
- # "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
- # "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
- # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"
- ]
- # 爬虫的IP代理
- DOWNLOADER_IP_PROXY = 'https://api.xiaoxiangdaili.com/ip/get?appKey=1114815955776983040&appSecret=M4RS3NpN&cnt=2&wt=json&method=http&city=&province='
- # 区名
- _QU = 'jimo'
- # 面积
- _AREA = '100_300'
- # 房租
- _MONEY = '0_10000'
- # 爬取的起始URL
- START_URL = f'https://qd.58.com/{_QU}/shangpucz/pn1/?area={_AREA}&huansuanyue={_MONEY}'
- # 持久化方式
- STORE_METHOD = 'csv'
- # csv文件夹
- CSV_DIR = 'output_csv'
- # csv文件名
- CSV_FILENAME = f'{_QU}_{_AREA}_{_MONEY}_{int(time.time() * 1000)}.csv'
- # 高德web服务API-key
- GAODE_KEY = '515a64d5324a70ba9c5a95f9539370ec'
- # 量化-半径
- BANJING = '750'
- # 量化-住宅个数
- ZHUZHAI_COUNT = 3
- # 量化-学校个数
- XUEXIAO_COUNT = 2
- # 量化-写字楼个数
- XIEZILOU_COUNT = 1
- # 量化-竞品门店个数
- JINGPIN_COUNT = 2
- # 量化-连锁品牌个数
- LIANSUO_COUNT = 2
- # 量化-连锁品牌名
- LIANSUO_LIST = ['好想来', '萨么', '丹香', '赵一鸣', '零食很忙', '糖巢', '大家乐', '元祖', '米兰西饼', '85度C', '幸福西饼', '好利来']
|
