爬取蝉妈妈数据平台商品数据

本文旨在交流学习，勿作他用，否则后果自负
环境 linux+pycharm+anaconda

import json
import csv
import requests
from usere_agent import UA
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


headers = {
 'Accept': '*/*',
 'Accept-Encoding': 'gzip, deflate, br',
 'Accept-Language': 'zh-CN,zh;q=0.9',
 'Connection': 'keep-alive',
 'Content-Length': '238',
 'Content-Type': 'application/x-www-form-urlencoded',
 'Host': 'api-service.chanmama.com',
 'Origin': 'https://www.chanmama.com',
 'Referer': 'https://www.chanmama.com/promotionRank',
 'User-Agent': UA
}

cl_url=["女装","男装","美妆护理","鞋包饰品","日用百货","母婴玩具","食品生鲜","运动户外","鲜花家纺","宠物农资","汽车配件","手机数码","生活家电","家装建材","其他",]



url = 'https://api-service.chanmama.com/v1/product/search'
for i in cl_url:
    for j in range(1, 1000):
        data1 = {'keyword': '', 'keyword_type': '', 'page': '{}'.format(j), 'price': '', 'size': '100',
                'filter_coupon': '0', 'is_aweme_goods': '0', 'tb_max_commission_rate': '', 'day_pv_count': '',
                'day_order_count': '', 'cat': "{}".format(i), 'platform': '', 'sort': "day_order_count",
                'order_by': "desc",
                }
        d = requests.post(url=url, headers=headers, data=(json.dumps(data1)), verify=False).json()
        # print(d)
        try:
            if d:
                data_list = d['data']['list']
                #print(data_list)

                if data_list:
                    for k in data_list:
                        img = k['image']  # 图片
                        #print(img)
                        title = k['title']  # 商品名
                        brand = k["brand"]  # 品牌
                        u = k["url"]  # 链接
                        market_price = k["market_price"]  # 市场价格
                        cat = k['cat']  # 分类
                        shop_name = k["shop_name"]  # 店铺名
                        sales = k["sales"]  # 全网月销量
                        tb_max_commission_rate = k["tb_max_commission_rate"]  # 佣金比例
                        day_pv_count = k["day_pv_count"]  # 昨日浏览量
                        conversion_rate = k["conversion_rate"]  # 昨日转化率
                        day_order_count = k["day_order_count"]  # 昨日销量
                        tb_coupon_price = k["tb_coupon_price"]  # 优惠价
                        print(cat, shop_name, brand, title, img, market_price, sales, tb_max_commission_rate,
                              day_pv_count, conversion_rate, day_order_count, tb_coupon_price, u)

                        with open('/media/liu/_dde_data/project/spider/供应商/cmm_data/' + cat + '.csv', 'a+') as f:
                            f_csv = csv.writer(f)
                            f_csv.writerow([cat,shop_name,brand,title,img,market_price,sales,tb_max_commission_rate,day_pv_count,conversion_rate,day_order_count,tb_coupon_price,u])
                else:
                    continue


        except Exception as e:
             continue