#!usr/bin/env python
# -*- coding:utf-8 -*-
"""
@author: dell
@file: class_washing.py
@time: 2021/03/26
@desc:
"""
import pandas as pd
import re
import numpy as np

# category为产品类型
# a是同一品牌或父品牌下产品数量占产品总数量的百分比，作为阈值，a越大，有异常的产品越多；b和a一样，只是用于产品数据类型和参数
# c_list是产品参数中，数据类型较为统一的参数 在excel列名中的位置，从0开始，必须是一个list


def class_washing(category, filepath, c_list,a=0.02, b=0.01):
    df = pd.read_excel(filepath,converters = {'产品编码':str})
    df.drop(columns='Unnamed: 0', axis=1, inplace=True)
    # 检测每列数据中 同一数据类型的产品数量少于产品总数量的b 的产品index.(可优化成字典形式存储读取)
    dtype_minor_dict = {}
    for col in df.columns:
        type_list = {}
        valid_index = df.index[(df[col] != '暂无数据') == ((df[col] != '无参数，需补充') == (df[col].notnull()))]
        for i in valid_index:
            data_type = type(df.loc[i, col])
            if data_type not in type_list:
                type_list[data_type] = 1
            elif data_type in type_list:
                type_list[data_type] += 1
        for data_type_i in type_list:
            if type_list[data_type_i] < len(valid_index) * b:
                for i in valid_index:
                    if type(df.loc[i][col]) == data_type_i:
                        if i in dtype_minor_dict.keys():
                            dtype_minor_dict[i] += ' '+col
                        elif i not in dtype_minor_dict.keys():
                            dtype_minor_dict[i] = col

    dtype_minor=[]
    dtype_minor.extend(dtype_minor_dict.keys())

    # 在检测完产品数据类型后，将所有数据类型转换为string
    for col in df.columns:
        df[col] = df[col].astype(str)

    # 检测产品类型错误的产品，和产品名称中不带有产品类型的产品。由于代码简单，就放在一起了
    wrong_class = []
    not_in_name = []
    for i in df.index:
        if df.loc[i, '产品类别'] != category:
            wrong_class.append(i)
        if category not in df.loc[i, '产品名称']:
            not_in_name.append(i)

    # 检测产品父品牌中品牌出现次数小于产品总数的a的产品
    father_brand_minor = []
    father_brand_list = []
    col='产品父品牌'
    valid_df=df.loc[df.index[(df[col] != '暂无数据') == ((df[col] != '无参数，需补充') == (df[col].notnull()))]]
    father_brand_num = valid_df.groupby(by='产品父品牌')['产品编码'].count()  # 之所以用产品编码来计数是因为产品编码肯定不会有缺失值
    father_num_list = [x for x in father_brand_num.unique() if x < len(valid_df.index)*a]
    for i in father_brand_num.index:  # i 就是产品父品牌
        if father_brand_num.loc[i] in father_num_list:  # father_brand_num.loc[i] 就是该父品牌出现次数
            father_brand_list.append(i)

    for i in valid_df.index:
        if valid_df.loc[i, '产品父品牌'] in father_brand_list:
            father_brand_minor.append(i)

    # 检测产品品牌中品牌出现次数少的产品
    brand_minor = []
    brand_list = []
    col = '产品品牌'
    valid_df = df.loc[df.index[(df[col] != '暂无数据') == ((df[col] != '无参数，需补充') == (df[col].notnull()))]]
    brand_num = valid_df.groupby(by='产品品牌')['产品编码'].count()  # 同上
    num_list = [x for x in brand_num.unique() if x < len(valid_df.index)*a]
    for i in brand_num.index:
        if brand_num.loc[i] in num_list:
            brand_list.append(i)

    for i in valid_df.index:
        if valid_df.loc[i, '产品品牌'] in brand_list:
            brand_minor.append(i)

    # 检测产品参数列中数据长度离该列平均数据长度2*std之外的产品，7是第一个产品参数列，-2是质保时间，-1是产品型号
    length_minor_dict = {}
    for col in df.columns[7:-2]:
        col_length = []
        valid_index = df.index[(df[col] != '暂无数据') == ((df[col] != '无参数，需补充') == (df[col].notnull()))]
        for i in valid_index:
            col_length.append(len(df.loc[i, col]))
        std = np.array(col_length).std()
        mean = np.array(col_length).mean()
        for counter, length in enumerate(col_length):
            if length < mean - 2 * std or length > mean + 2 * std:
                # length_minor_dict[valid_index[counter]]=col
                index=valid_index[counter]
                if index in length_minor_dict.keys():
                    length_minor_dict[index] += ' ' + col
                elif index not in length_minor_dict.keys():
                    length_minor_dict[index] = col
    length_minor=[]
    length_minor.extend(length_minor_dict.keys())
    # 检测产品参数列数据格式小于总数量的b的产品
    format_minor_dict = {}
    for col in df.columns[7:-2]:
        counter_dict = {}
        valid_index = df.index[(df[col] != '暂无数据') == ((df[col] != '无参数，需补充') == (df[col].notnull()))]
        for i in valid_index:
            counter_list = []
            k = df.loc[i, col]
            is_str = re.findall(r'[A-Za-z]+', k)
            is_dig = re.findall(r'[0-9]+', k)
            is_special = re.findall(r'\W+', k)
            is_chinese = re.findall(r'[\u4e00-\u9fa5]+', k)
            if is_str:
                counter_list.append('str')
            if is_dig:
                counter_list.append('dig')
            if is_special:
                counter_list.append('special')
            if is_chinese:
                counter_list.append('chinese')
            combined = ''.join(counter_list)
            if combined not in counter_dict:
                counter_dict[combined] = [i]
            elif combined in counter_dict:
                counter_dict[combined].append(i)

        for keys in counter_dict:
            if len(counter_dict[keys]) < len(valid_index)*b:
                for keys_index in counter_dict[keys]:
                    if keys_index in format_minor_dict.keys():
                        format_minor_dict[keys_index] += ' ' + col
                    elif keys_index not in format_minor_dict.keys():
                        format_minor_dict[keys_index] = col


        format_minor=[]
        format_minor.extend(format_minor_dict.keys())
        # length_record = []
        # for keys in counter_dict:
        #     if not length_record:
        #         length_record.append([len(counter_dict[keys]), counter_dict[keys]])
        #     elif len(counter_dict[keys]) < length_record[0][0]:
        #         length_record[0] = [len(counter_dict[keys]), counter_dict[keys]]
        #
        # format_minor += length_record[0][1]

    # 接下来是针对扫描仪的部分.对于特定产品，还可以从每个产品参数中选出少数派。如果知道易混淆的产品类型，还要特意加上挑取易混淆产品类型的代码
    # 对于产品名称中没有“扫描仪”的，如果没有“高拍仪”就挑出来
    not_in_name2 = []
    for i in not_in_name:
        if '高拍仪' not in df.loc[i,'产品名称']:
            not_in_name2.append(i)

    # 对于产品参数中，数据类型较少的参数，其中如果有数量小于产品总数量的b的，挑出来
    character_minor_dict = {}
    for col_i in df.columns[c_list]:
        tempo_list = []
        tempo_list2 = []
        valid_df=df.loc[df.index[(df[col_i] != '暂无数据') == ((df[col_i] != '无参数，需补充') == (df[col_i].notnull()))]]
        cha_num = valid_df.groupby(by=col_i)['产品编码'].count()
        num_list = [x for x in cha_num.unique() if x < len(valid_df.index)*b]
        for i in cha_num.index:
            if cha_num.loc[i] in num_list:
                tempo_list.append(i)
        for i in valid_df.index:
            if valid_df.loc[i, col_i] in tempo_list:
                tempo_list2.append(i)
        character_minor_dict[col_i] = tempo_list2

    character_minor = []
    for keys_i in character_minor_dict:
        character_minor.extend(character_minor_dict[keys_i])

    # 将挑出的可疑数据整合到一起 (wrong_class没加，因为里面的肯定不对）
    index_minor = []
    index_minor.extend(wrong_class)
    index_minor.extend(format_minor)
    index_minor.extend(length_minor)
    index_minor.extend(brand_minor)
    index_minor.extend(father_brand_minor)
    index_minor.extend(not_in_name2)
    index_minor.extend(dtype_minor)
    index_minor.extend(character_minor)
    index_minor = set(index_minor)

    final_df = pd.DataFrame(np.zeros((len(index_minor), 8)), index=list(index_minor),
                            columns=['计数', '产品类型异常', '产品名称异常', '父品牌异常', '品牌异常', '数据类型异常', '数据格式异常', '数据长度异常'])
    for i in index_minor:
        count = 0
        if i in wrong_class:
            count += 1
            final_df.loc[i, '产品类型异常'] = 1
        if i in format_minor:
            count += len(format_minor_dict[i].split())  # 如果该行数据有多列数据格式异常，就要加多次,
            final_df.loc[i, '数据格式异常'] = format_minor_dict[i]  # 但其中数据是空格分割的字符串，所以用split
        if i in length_minor:
            count += len(length_minor_dict[i].split())
            final_df.loc[i, '数据长度异常'] = length_minor_dict[i]
        if i in brand_minor:
            count += 1
            final_df.loc[i, '品牌异常'] = 1
        if i in father_brand_minor:
            count += 1
            final_df.loc[i, '父品牌异常'] = 1
        if i in not_in_name2:
            count += 1
            final_df.loc[i, '产品名称异常'] = 1
        if i in dtype_minor:
            count += len(dtype_minor_dict[i].split())
            final_df.loc[i, '数据类型异常'] = dtype_minor_dict[i]
        for keys_i in character_minor_dict:
            if i in character_minor_dict[keys_i]:
                final_df.loc[i, keys_i + '异常'] = 1
                count += 1
            else:
                final_df.loc[i, keys_i + '异常'] = 0
        final_df.loc[i, '计数'] = count

    final_df = pd.merge(final_df, df, how='left', left_index=True, right_index=True)
    final_df = final_df.sort_values(by='计数', ascending=False)
    final_df.to_excel("./class_filter.xlsx")


if __name__ == '__main__':
    category='激光打印机'
    filepath="E:\\ZDZC\\激光打印机参数确认.xlsx"
    c_list=[6,7,-4,-3]
    # category = '扫描仪'
    # filepath="E:\\ZDZC\\扫描仪参数确认.xlsx"
    # c_list=[7,8,9]
    class_washing(category, filepath, c_list)
