#!usr/bin/env python
# -*- coding:utf-8 -*-


import pandas as pd
import re
import xlsxwriter
import numpy as np
import pymssql

def brand_washing_special(filepath, sheet_name, category):
    df = pd.read_excel(filepath, sheet_name=sheet_name, converters={'产品编码': str})
    conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311', password='zgcprice20200628',
                                  database='ZI_NEW', autocommit=True)
    cursor_zi_new = conn_zi_new.cursor()
    cursor_zi_new.execute(f"select col_value from data_washing_external where category_name='通用' and col_name='异常数据名称'")
    invalid_list_fetch = cursor_zi_new.fetchall()
    invalid_list = []
    for invalid_tuple in invalid_list_fetch:
        invalid_list.append(invalid_tuple[0])
    # 处理缺失值
    valid_index = []
    if category == '复印纸':
        for i in df.index:
            if df.loc[i, '产品品牌'] not in invalid_list and df.loc[i, '产品系列（SPU）'] not in invalid_list:
                valid_index.append(i)
        result = df.loc[valid_index, ['产品品牌', '产品系列（SPU）']]
        # 将df数据格式转为字符串
        result = result.apply(lambda x: x.astype(str))
        # def xilie_clean(x):
        #     x = x.strip().replace('系列','').upper()
        #     ch = re.findall(r'([\u4e00-\u9fa5]+)', x)
        #     en = re.findall(r'[0-9a-zA-Z]+', x)
        #     if ch and en:
        #         return
        result['产品系列（SPU）'] = result['产品系列（SPU）'].apply(lambda x: x.strip().replace('系列', '').upper())
        result_groupby = dict(list(result.groupby('产品品牌')))
        brand_dict = {}
        for brand in result_groupby.keys():
            xilie_list = result_groupby[brand]['产品系列（SPU）'].unique().tolist()
            try:
                xilie_list.remove('彩色复印纸')
            except:
                pass
            brand_dict[brand] = xilie_list
    else:
        for i in df.index:
            if df.loc[i, '产品品牌'] not in invalid_list and df.loc[i, '*产品系列'] not in invalid_list:
                valid_index.append(i)
        result = df.loc[valid_index,['产品品牌','*产品系列']]
        # 将df数据格式转为字符串
        result = result.apply(lambda x: x.astype(str))
        # def xilie_clean(x):
        #     x = x.strip().replace('系列','').upper()
        #     ch = re.findall(r'([\u4e00-\u9fa5]+)', x)
        #     en = re.findall(r'[0-9a-zA-Z]+', x)
        #     if ch and en:
        #         return
        result['*产品系列'] = result['*产品系列'].apply(lambda x: x.strip().replace('系列','').upper())
        result_groupby = dict(list(result.groupby('产品品牌')))
        brand_dict={}
        for brand in result_groupby.keys():
            xilie_list = result_groupby[brand]['*产品系列'].unique().tolist()
            brand_dict[brand] = xilie_list

    related_brand=[]
    tested_brand = []
    repeated_xilie = []

    for brand_i in brand_dict.keys():
        for brand_j in brand_dict.keys():
            if brand_i == brand_j or {brand_i, brand_j} in tested_brand:
                continue
            temp_repeated_xilie=[]
            for xilie in brand_dict[brand_i]:
                if xilie in brand_dict[brand_j]:
                    temp_repeated_xilie.append(xilie)
            if temp_repeated_xilie:
                repeated_xilie.append(temp_repeated_xilie)
                related_brand.append([brand_i, brand_j])
            tested_brand.append({brand_i, brand_j})
    #输出到excel里
    workbook = xlsxwriter.Workbook('./brand_filter.xlsx')
    bold_format = workbook.add_format({'bold': True})

    worksheet = workbook.add_worksheet(name='Sheet1')
    worksheet.write('A1', '品牌A', bold_format)
    worksheet.write('B1', '品牌B', bold_format)
    worksheet.write('C1', '正确品牌', bold_format)
    col = 0
    row = 1
    for brand_list in related_brand:
        for brand in brand_list:
            worksheet.write_string(row, col, brand)
            col += 1
        row += 1
        col = 0

    worksheet2 = workbook.add_worksheet(name='Sheet2')
    worksheet2.write('A1', '品牌A', bold_format)
    worksheet2.write('B1', '品牌B', bold_format)
    worksheet2.write('C1', '重复系列', bold_format)
    row = 1
    col = 0
    for i in range(len(related_brand)):
        worksheet2.write_string(row, col, related_brand[i][0])
        col += 1
        worksheet2.write_string(row, col, related_brand[i][1])
        col += 1
        worksheet2.write_string(row, col, f"{repeated_xilie[i]}")
        row += 1
        col = 0

    worksheet3 = workbook.add_worksheet(name='Sheet3')
    row = 0
    col = 0
    for key in brand_dict.keys():
        worksheet3.write_string(row, col, key)
        col += 1
        worksheet3.write_string(row, col, f"{brand_dict[key]}")
        col = 0
        row += 1

    workbook.close()













def brand_washing(filepath,thre=0.5,inner_thre=0.5,a=1,sheet_name=0):
    # filepath:文件路径，thre为两个品牌下型号重合率阈值，inner_thre为两个品牌下某条型号内关键词重合率阈值，a为权重调整，sheet_name为表单名
    df = pd.read_excel(filepath, sheet_name=sheet_name, converters = {'产品编码':str})
    category = df.loc[1,'产品类别']
    wrong_category_list = ['台式机', '笔记本', '一体电脑', '复印纸']
    if category in wrong_category_list:
        brand_washing_special(filepath, sheet_name, category)
        return
    conn_zi_new = pymssql.connect(host='123.56.115.207', user='zgcprice3311', password='zgcprice20200628',
                                  database='ZI_NEW', autocommit=True)
    cursor_zi_new = conn_zi_new.cursor()
    cursor_zi_new.execute(f"select col_value from data_washing_external where category_name='通用' and col_name='异常数据名称'")
    invalid_list_fetch = cursor_zi_new.fetchall()
    invalid_list = []
    for invalid_tuple in invalid_list_fetch:
        invalid_list.append(invalid_tuple[0])
    # 处理缺失值
    valid_index=[]
    for i in df.index:
        if df.loc[i,'产品品牌'] not in invalid_list and df.loc[i,'*产品型号'] not in invalid_list:
            valid_index.append(i)
    result = df.loc[valid_index]
    # 将df数据格式转为字符串
    for i in result.columns:
        result[i] = result[i].astype(str)

    # 创建一个字典，key是品牌名，values是一个列表，列表元素是集合，集合元素是品牌下产品型号的关键字
    standard_library = dict()
    # 将df按品牌分类，按品牌提取每个型号的关键字，放入字典
    brand_type = result.groupby('产品品牌')
    for brand in brand_type:
        result_unique = brand[1]['*产品型号'].unique()  # result_unique此时是array，元素是一个品牌名下的型号
        for j in range(len(result_unique)):
            result_unique[j] = result_unique[j].upper().strip()
        result_unique = pd.DataFrame(result_unique)[0].unique()  # 全部变为大写，将大小写归一,result_unique此时是array

        temp_list = []  # 创建一个临时列表用于收集同一品牌下每个型号提取的关键字集合
        for k in result_unique:
            pre_num = re.search(r'([A-Za-z]{0,4})\W?\d+', k)  # pre_num为数字前的关键字
            if pre_num:
                pre_num = [pre_num.group(1)]
            if not pre_num:
                pre_num = []

            num = re.search(r'(\d+)', k)  # num为数字关键字
            if num:
                num = [num.group(1)]  # 如果连续数字超过1处，我们只提取第一处，这个可能有点问题但目前还不知道怎么办
            if not num:  # 如果没有数字，就比较英文单词
                num = re.findall(r'[A-Za-z]+', k)

            pos_num = re.findall(r'\d+([A-Za-z]+)?\W?([0-9]+)?\W?([A-Za-z]+)?\W?([0-9]+)?\W?([A-Za-z]+)?\W?([0-9]+)?\W?([A-Za-z]+)?',
                                 k)  # pos_num为数字后的关键字
            if pos_num:
                pos_num = list(pos_num[0])
                if '升级版' in k:
                    pos_num += '升级版'
                if '专业版' in k:
                    pos_num += '专业版'
                if '教育版' in k:
                    pos_num += '教育版'
                if '+' in k:
                    pos_num += '+'

            combined = pre_num + num + pos_num  # 将关键字列表合并
            while '' in combined:
                combined.remove('')  # 删除空的元素
            if combined:
                if set(combined) not in temp_list:
                    temp_list.append(set(combined))  # 每一条型号的关键词列表变成一个集合，添加到临时列表中
        standard_library[brand[0]] = temp_list  # 将临时列表添加到字典中对应的品牌名下

    # 第一种筛选方法：通过比对两个品牌下每个型号关键字重合度来判断两个品牌是否相似
    related_brand1 = []  # 用于接收相似的品牌名
    tested_brand = []  # 用于接收已经测试过的品牌名
    for brand_i in standard_library:  # brand_i为品牌名
        for brand_j in standard_library:
            if brand_i != brand_j:  # 相同的品牌名不对比
                if set([brand_i, brand_j]) not in tested_brand:  # 对比过的品牌名不对比
                    accuracy_i = 0  # 准确率，每个品牌一更新
                    accuracy_j = 0
                    related_version=[]
                    for version_i in standard_library[brand_i]:  # version_i为品牌下一个产品型号关键字组成的集合
                        for version_j in standard_library[brand_j]:
                            inner_accuracy_i = 0  # 内部准确率，每个型号集合一更新
                            inner_accuracy_j = 0
                            for word_i in version_i:  # word_i为集合内的一个关键字
                                if word_i in version_j:
                                    inner_accuracy_i += (1 / (len(version_i) + a / len(version_i)))  # a为调整权重参数，默认为1
                                    inner_accuracy_j += (1 / (len(version_i) + a / len(version_j)))
                            count = 0
                            if inner_accuracy_i >= inner_thre:  # 如果两个集合内的关键字重合率达到内阈值，则增加对应准确率
                                accuracy_i += (
                                            1 / (len(standard_library[brand_i]) + a / len(standard_library[brand_i])))
                                count += 1
                            if inner_accuracy_j >= inner_thre:
                                accuracy_j += (
                                            1 / (len(standard_library[brand_i]) + a / len(standard_library[brand_i])))
                                count += 1

                            if count != 0:
                                related_version.append([' '.join(version_i), ' '.join(version_j)])
                                break  # 如果有增加准确率，则代表两个集合已匹配成功，不再继续往下找，换brand_i下一个型号集合匹配
                    if accuracy_i >= thre or accuracy_j >= thre:  # 如果任一准确率达到阈值，在相近品牌列表中增加这两个品牌
                        tempo_list=[]
                        tempo_list.extend([brand_i, brand_j, max(accuracy_i, accuracy_j)])
                        tempo_list.extend(related_version)
                        related_brand1.append(tempo_list)

                tested_brand.append(set([brand_i, brand_j]))  # 测试后，在已测试列表中增加测试过的品牌

    # 第二种方法，对比两个品牌名称的关键字，只要有一个重合，就算作两个品牌相似
    brand_kw = {}  # 用于接收品牌名提取的关键字
    for i in brand_type:
        temp_list1 = re.findall(r'([\u4e00-\u9fa5]+)', i[0].upper())  # 提取汉字
        while '新建品牌' in temp_list1:
            temp_list1.remove('新建品牌')  # 去除‘新建品牌’
        temp_list2 = re.findall(r'[A-Za-z]+\W?[A-Za-z]+', i[0].upper())  # 提取英文单词
        brand_kw[i[0]] = temp_list1 + temp_list2

    related_brand2 = []  # 相似品牌
    tested_brand = []  # 测试过的品牌
    for brand_i in brand_kw:
        for brand_j in brand_kw:
            if brand_i != brand_j:
                if set([brand_i, brand_j]) not in tested_brand:
                    for word_i in brand_kw[brand_i]:
                        if word_i in brand_kw[brand_j]:
                            related_brand2.append([brand_i, brand_j])
                        else:
                            continue
                        break  # 跳过两层循环，只要有一个关键字相同就代表品牌类似
                    tested_brand.append(set([brand_i, brand_j]))

    # 第三种方法，对比品牌名和另一个品牌型号关键字，如果有一个重合，就算作两个品牌相似
    type_kw = {}  # 用于接收品牌型号提取的关键字
    for i in brand_type:
        result_unique = i[1]['*产品型号'].unique()  # 品牌型号组成的数组，数组内无重复元素
        for k in range(len(result_unique)):
            result_unique[k]=result_unique[k].upper().strip()
        result_unique = pd.DataFrame(result_unique)[0].unique()
        type_kw[i[0]] = result_unique

    related_brand3 = []  # 相似品牌

    for brand_i in brand_kw:
        for brand_j in type_kw:
            brand_list = []
            word_list = []
            if brand_i != brand_j:
                for word_i in brand_kw[brand_i]:
                    for word_j in type_kw[brand_j]:
                        if word_i in word_j:
                            brand_list.extend([brand_i, brand_j])
                            word_list.append([word_i, word_j])
            if brand_list:
                tempo_list=brand_list[:2]
                tempo_list.extend(word_list)
                related_brand3.append(tempo_list)

    #写入excel
    tempo_list = []  # 将所有的相似品牌，两两一对，写入集合，放入tempo_list
    method1=[]
    method2=[]
    method3=[]
    related_brand_list=[related_brand1,related_brand2,related_brand3]
    method_list=[method1,method2,method3]
    for i in range(len(related_brand_list)):
        for list_i in related_brand_list[i]:
            tempo_list.append(set(list_i[:2]))
            method_list[i].append(set(list_i[:2]))


    final_list = []  # final_list就是tempo_list的去重
    for item in tempo_list:
        if item not in final_list:
            final_list.append(item)

    method=[]
    for item in final_list:
        linshi_list=[]
        if item in method1:
            linshi_list.append('1')
        if item in method2:
            linshi_list.append('2')
        if item in method3:
            linshi_list.append('3')
        method.append(','.join(linshi_list))



    workbook = xlsxwriter.Workbook('./brand_filter.xlsx')
    bold_format = workbook.add_format({'bold': True})

    # 第一个worksheet
    worksheet = workbook.add_worksheet(name='Sheet1')
    worksheet.write('A1', '品牌A', bold_format)
    worksheet.write('B1', '品牌B', bold_format)
    worksheet.write('C1', '正确品牌', bold_format)
    worksheet.write('D1', '方法', bold_format)
    col = 0
    row = 1
    for index in range(len(final_list)):
        for brand_i in final_list[index]:
            worksheet.write_string(row, col, brand_i)
            col += 1
        col= 3
        worksheet.write_string(row,col,method[index])
        row += 1
        col = 0

    # for list_i in related_brand2:
    #     for brand_i in range(2):
    #         worksheet.write_string(row, col, list_i[brand_i])
    #         col += 1
    #     col = 3
    #     worksheet.write_string(row, col, '2')
    #     row += 1
    #     col = 0
    #
    # for list_i in related_brand3:
    #     for brand_i in range(2):
    #         worksheet.write_string(row, col, list_i[brand_i])
    #         col += 1
    #     col = 3
    #     worksheet.write_string(row, col, '3')
    #     row += 1
    #     col = 0

    # 第二个worksheet
    worksheet2 = workbook.add_worksheet(name='Sheet2')
    col = 0
    row = 0
    for list_i in related_brand1:
        for brand_i in list_i[:2]:
            worksheet2.write_string(row, col, brand_i)
            row += 1
        row+=1

    row=0

    for list_i in related_brand1:
        col = 1
        for version_list in list_i[3:]:
            for version_i in version_list:
                worksheet2.write_string(row, col, version_i)
                row += 1
            row -= 2
            col += 1
        row += 3


    col=0
    row_marker=row
    for list_i in related_brand3:
        for brand_i in list_i[:2]:
            worksheet2.write_string(row, col, brand_i)
            row += 1
        row+=1

    row=row_marker

    for list_i in related_brand3:
        col = 1
        for version_list in list_i[2:]:
            for version_i in version_list:
                worksheet2.write_string(row, col, version_i)
                row += 1
            row -= 2
            col += 1
        row += 3
    workbook.close()

if __name__ == '__main__':
    filepath = 'E:\\ZDZC\\Sourcetree_local\\公共代码\\复印纸参数确认.xlsx'
    brand_washing(filepath)

