#!usr/bin/env python
# -*- coding:utf-8 -*-
"""
@author: dell
@file: class_washing.py
@time: 2021/03/26
@desc:
"""
import pandas as pd
import re
import numpy as np
import pymssql
import time

# category为产品类型
# b是阈值，用于数据类型，数据格式；比如当某列数据类型占总数据量比例小于b时，该列拥有数据类型的数据索引将被作为异常数据返回


def class_washing(category, filepath, b=0.01):
    conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311', password='zgcprice20200628',
                                  database='ZI_NEW', autocommit=True)
    cursor_zi_new = conn_zi_new.cursor()
    # 一些数据写着‘暂无数据’，‘无参数，需补充’等替代值，这些替代值从外部表格被提取到invalid_list中
    cursor_zi_new.execute(f"select col_value from data_washing_external where category_name='通用' and col_name='异常数据名称'")
    invalid_list_fetch=cursor_zi_new.fetchall()
    invalid_list=[]
    for invalid_tuple in invalid_list_fetch:
        invalid_list.append(invalid_tuple[0])

    df = pd.read_excel(filepath,converters = {'产品编码':str})
    df.drop(columns='Unnamed: 0', axis=1, inplace=True)

    # 检测每列数据中 同一数据类型的产品数量少于产品总数量的b 的产品index. 数据类型的例子：string, float, int
    t1=time.time()
    print('开始检测数据类型')

    dtype_minor_dict = {}  # 该字典键为异常数据索引，值为异常数据类型出现的列名，值是字符串而非列表，多个列名由空格隔开
    #   #从外部表提取用于判断‘数据类型异常’的列，放入dtype_col_list
    cursor_zi_new.execute(f"select col_value from data_washing_external where category_name='{category}' and col_name='数据类型异常'")
    dtype_col_fetch=cursor_zi_new.fetchall()
    if not dtype_col_fetch:
        print('该类别无‘数据类型异常’列名，或者类别名输入有误，请查证数据库')
        return
    dtype_col_list=[]
    for dtype_tuple in dtype_col_fetch:
        dtype_col_list.append(dtype_tuple[0])
    #   #按列找
    for col in dtype_col_list:
        type_list = {}  # 键为该列出现的数据类型，值为该类型出现的次数
        valid_index = []  # 值为‘暂无数据’等替代值的数据被忽略，值为实际值的数据索引提取到valid_index
        for i in df.index:
            if df.loc[i,col] in invalid_list:
                continue
            valid_index.append(i)
            data_type = type(df.loc[i, col])
            if data_type not in type_list:
                type_list[data_type] = 1
            elif data_type in type_list:
                type_list[data_type] += 1
        for data_type_i in type_list:
            if type_list[data_type_i] < len(valid_index) * b:
                # 下面这一段可以优化，可以在type_list字典中存储每个数据类型对应的数值索引，先记下，优化阶段再改
                for i in valid_index:
                    if type(df.loc[i][col]) == data_type_i:
                        if i in dtype_minor_dict.keys():
                            dtype_minor_dict[i] += ' '+col
                        elif i not in dtype_minor_dict.keys():
                            dtype_minor_dict[i] = col

    #   #将dtype_minor_dict字典的键，也就是异常数据索引，放入dtype_minor
    dtype_minor=[]
    dtype_minor.extend(dtype_minor_dict.keys())

    # 在检测完产品数据类型后，将所有数据类型转换为string，因为后面的操作都是对于字符串的，如果有数据类型不是字符串就比较麻烦
    for col in df.columns:
        df[col] = df[col].astype(str)

    # 检测产品类型错误的产品，和产品名称中不带有产品类型的产品。由于代码简单，就放在一起了
    t2=time.time()
    print(t2-t1)
    print('开始检测错误类别和错误名称')
    wrong_class = []  # 类型错误数据索引
    not_in_name = []  # 名称中不带类型的数据索引
    for i in df.index:
        if df.loc[i, '产品类别'] != category:
            wrong_class.append(i)
        if category not in df.loc[i, '产品名称']:
            not_in_name.append(i)

    # 检测品牌中有哪些不在数据表中该类别下属品牌中
    t3=time.time()
    print(t3-t2)
    print('开始检测错误品牌')

    #   #提取类别id
    cursor_zi_new.execute(f"select id from p_category where name='{category}'")
    category_id=cursor_zi_new.fetchone()
    if not category_id:
        print('输入类别不在数据库中，请查证')
        return
    category_id = category_id[0]

    #   #找到该类别下的品牌的brand_id
    cursor_zi_new.execute(f"select brandid from p_spu where categoryid={category_id}")
    brand_id_fetchall=cursor_zi_new.fetchall()
    brand_id_list = []
    for brand_tuple in brand_id_fetchall:
        brand_id_list.append(brand_tuple[0])

    #   #从brand_id找到品牌名（为什么不直接从category_id找品牌名呢？问得好，没有这个表）
    brand_name_list = []
    for brand_id in brand_id_list:
        cursor_zi_new.execute(f"select name from p_brand where id={brand_id}")
        brand_name_fetch=cursor_zi_new.fetchone()
        if brand_name_fetch:
            brand_name_list.append(brand_name_fetch[0].strip("'"))

    #   #如果哪条数据品牌没在品牌列表中，作为异常值返回
    wrong_brand=[]
    for i in df.index:
        if df.loc[i]['产品品牌'] not in brand_name_list:
            wrong_brand.append(i)

    # 检测产品参数列中数据长度离该列平均数据长度2*std之外的产品
    t4=time.time()
    print(t4-t3)
    print('开始检测错误长度')

    length_minor_dict = {}
    #   #从外部表找到用于数据长度判断的列名
    cursor_zi_new.execute(f"select col_value from data_washing_external where category_name='{category}' and col_name='数据长度异常'")
    length_col_fetch = cursor_zi_new.fetchall()
    if not length_col_fetch:
        print('该类别无‘数据长度异常’列名，或者类别名输入有误，请查证数据库')
        return
    length_col_list = []
    for length_tuple in length_col_fetch:
        length_col_list.append(length_tuple[0])

    #   #按列找
    for col in length_col_list:
        col_length = []  # 包含该列每条数据的长度
        valid_index=[]  # 同上
        for i in df.index:
            if df.loc[i, col] in invalid_list:
                continue
            valid_index.append(i)
            col_length.append(len(df.loc[i, col]))
        if col_length:  # 有时候一列数据全都是无效数据所以加个条件
            std = np.array(col_length).std()
            mean = np.array(col_length).mean()
            for counter, length in enumerate(col_length):
                if length < mean - 2 * std or length > mean + 2 * std:
                    index=valid_index[counter]  # valid_index和col_length是一一对应的
                    if index in length_minor_dict.keys():
                        length_minor_dict[index] += ' ' + col
                    elif index not in length_minor_dict.keys():
                        length_minor_dict[index] = col

    length_minor=[]
    length_minor.extend(length_minor_dict.keys())

    # 检测产品参数列数据格式小于总数量的b的产品
    t5=time.time()
    print(t5-t4)
    print('开始检测错误数据格式')

    #   #从外部表中提取用于数据格式判断的列名
    format_minor_dict = {}
    cursor_zi_new.execute(
        f"select col_value from data_washing_external where category_name='{category}' and col_name='数据格式异常'")
    format_col_fetch = cursor_zi_new.fetchall()
    if not format_col_fetch:
        print('该类别无‘数据格式异常’列名，或者类别名输入有误，请查证数据库')
        return
    format_col_list = []
    for format_tuple in format_col_fetch:
        format_col_list.append(format_tuple[0])

    #   #按列找
    for col in format_col_list:
        counter_dict = {}
        valid_index = []
        for i in df.index:
            if df.loc[i, col] in invalid_list:
                continue
            valid_index.append(i)
        for i in valid_index:
            counter_list = []
            k = df.loc[i, col]
            is_str = re.findall(r'[A-Za-z]+', k)
            is_dig = re.findall(r'[0-9]+', k)
            is_special = re.findall(r'\W+', k)
            is_chinese = re.findall(r'[\u4e00-\u9fa5]+', k)
            if is_str:
                counter_list.append('str')
            if is_dig:
                counter_list.append('dig')
            if is_special:
                counter_list.append('special')
            if is_chinese:
                counter_list.append('chinese')
            combined = ''.join(counter_list)
            if combined not in counter_dict:
                counter_dict[combined] = [i]
            elif combined in counter_dict:
                counter_dict[combined].append(i)

        for keys in counter_dict:
            if len(counter_dict[keys]) < len(valid_index)*b:
                for keys_index in counter_dict[keys]:
                    if keys_index in format_minor_dict.keys():
                        format_minor_dict[keys_index] += ' ' + col
                    elif keys_index not in format_minor_dict.keys():
                        format_minor_dict[keys_index] = col

    format_minor=[]
    format_minor.extend(format_minor_dict.keys())

    # 对于产品名称的第二次筛选，比如在扫描仪中，对于产品名称中没有“扫描仪”的，如果没有“高拍仪”就挑出来
    cursor_zi_new.execute(
        f"select col_value from data_washing_external where category_name='{category}' and col_name='产品名称异常'")
    name_col_fetch = cursor_zi_new.fetchall()
    name_col_list = []
    for name_tuple in name_col_fetch:
        name_col_list.append(name_tuple[0])
    not_in_name2 = []
    for i in not_in_name:
        for special_name in name_col_list:
            if special_name in df.loc[i,'产品名称']:
                break
        else:
            not_in_name2.append(i)

    # 对于标准产品参数中，如果有数据不在标准字典中的，挑出来
    t6=time.time()
    print(t6-t5)
    print('开始检测标准参数')

    # cursor_zi_new.execute(
    #     f"select col_value from data_washing_external where category_name='{category}' and col_name='标准参数异常'")
    # stdparam_col_fetch = cursor_zi_new.fetchall()
    # if not stdparam_col_fetch:
    #     print('该类别无‘标准参数异常’列名，或者类别名输入有误，请查证数据库。代码继续运行')
    # stdparam_col_list = []
    # for stdparam_tuple in stdparam_col_fetch:
    #     stdparam_col_list.append(stdparam_tuple[0])

    cursor_zi_new.execute(
        f"select subtitle, identy from vw_property where categoryid={category_id} and identy!='{0000}'"
    )
    subtitle_fetch=cursor_zi_new.fetchall()
    if not subtitle_fetch:
        print('该类别在vw_property中没有identy不等于0000的数据，请查证数据库')
        return
    stdparam_col_list=[]
    for subtitle_tuple in subtitle_fetch:
        if subtitle_tuple[1][2]=='1':  # identy第三个值为1代表是标准项
            stdparam_col_list.append(subtitle_tuple[0])

    if not stdparam_col_list:
        print('该类别无标准参数项, 请去库里查证，代码继续运行')

    character_minor_dict = {}
    for col_i in stdparam_col_list:
        temp_list = []
        cursor_zi_new.execute(f"select stdvalue from ShuJuZiDian_Cfg where categoryname='{category}' and subtitle='{col_i.strip('*')}'")
        standard_value_fetchall=cursor_zi_new.fetchall()
        if not standard_value_fetchall:
            print(f"{col_i.strip('*')} 不在 ShuJuZiDian_Cfg,请检查。该参数项在此次运行中未被采用,代码继续运行")
            continue

        standard_value=[]
        for std_tuple in standard_value_fetchall:
            standard_value.append(std_tuple[0])
        # print(f"{col_i},{len(standard_value_fetchall)}")
        for i in df.index:
            value_col=df.loc[i, '*'+col_i]
            if value_col in invalid_list:
                continue
            if value_col not in standard_value:
                temp_list.append(i)

        character_minor_dict[col_i] = temp_list

    character_minor = []
    for keys_i in character_minor_dict:
        character_minor.extend(character_minor_dict[keys_i])

    t7=time.time()
    print(t7-t6)
    print('开始整合数据')
    # 将挑出的可疑数据整合到一起
    index_minor = []
    index_minor.extend(wrong_class)
    index_minor.extend(format_minor)
    index_minor.extend(length_minor)
    index_minor.extend(wrong_brand)
    index_minor.extend(not_in_name2)
    index_minor.extend(dtype_minor)
    index_minor.extend(character_minor)
    index_minor = set(index_minor)

    final_df = pd.DataFrame(np.zeros((len(index_minor), 7)), index=list(index_minor),
                            columns=['计数', '产品类别异常', '产品名称异常', '品牌异常', '数据类型异常', '数据格式异常', '数据长度异常'])

    weight_list=[]
    weight_colname_list=['产品类别异常权重','数据格式异常权重','数据长度异常权重','品牌异常权重','产品名称异常权重','数据类型异常权重','标准参数异常权重']
    for col_name in weight_colname_list:
        cursor_zi_new.execute(
            f"select col_value from data_washing_external where category_name='{category}' and col_name='{col_name}'")
        weight_fetch=cursor_zi_new.fetchone()
        if weight_fetch:
            weight_list.append(weight_fetch[0])
        else:
            weight_list.append(1.0)

    w_class = float(weight_list[0])
    w_format = float(weight_list[1])
    w_length = float(weight_list[2])
    w_brand = float(weight_list[3])
    w_name = float(weight_list[4])
    w_dtype = float(weight_list[5])
    w_stdparam = float(weight_list[6])

    for i in index_minor:
        count = 0
        if i in wrong_class:
            count += w_class
            final_df.loc[i, '产品类别异常'] = 1
        if i in format_minor:
            count += len(format_minor_dict[i].split()) * w_format  # 如果该行数据有多列数据格式异常，就要加多次,
            final_df.loc[i, '数据格式异常'] = format_minor_dict[i]  # 但其中数据是空格分割的字符串，所以用split
        if i in length_minor:
            count += len(length_minor_dict[i].split()) * w_length
            final_df.loc[i, '数据长度异常'] = length_minor_dict[i]
        if i in wrong_brand:
            count += w_brand
            final_df.loc[i, '品牌异常'] = 1
        if i in not_in_name2:
            count += w_name
            final_df.loc[i, '产品名称异常'] = 1
        if i in dtype_minor:
            count += len(dtype_minor_dict[i].split()) * w_dtype
            final_df.loc[i, '数据类型异常'] = dtype_minor_dict[i]
        for keys_i in character_minor_dict:
            if i in character_minor_dict[keys_i]:
                final_df.loc[i, keys_i + '异常'] = 1
                count += w_stdparam
            else:
                final_df.loc[i, keys_i + '异常'] = 0
        final_df.loc[i, '计数'] = count

    final_df = pd.merge(final_df, df, how='left', left_index=True, right_index=True)
    final_df = final_df.sort_values(by='计数', ascending=False)
    final_df.to_excel("./class_filter.xlsx")


if __name__ == '__main__':
    category='激光打印机'
    filepath="E:\\ZDZC\\打印机参数确认.xlsx"
    class_washing(category, filepath)
