2020 Big Contest - 3

from google.colab import drive

drive.mount('/content/drive')

import pandas as pd
import numpy as np
import random
import sys
import re
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import RobustScaler

from keras.layers import Dense, Activation, Flatten, Conv1D, MaxPooling1D, AveragePooling1D, GlobalMaxPooling1D, Conv1DTranspose, concatenate, Input, add, Dropout, BatchNormalization
from tensorflow.keras import Model

# 숫자 표기법 변환
pd.set_option('display.float_format', '{:.2f}'.format)

# 한글 깨짐 방지
plt.rc('font', family='Malgun Gothic')

# 도화지 크기 설정
plt.rcParams['figure.figsize'] = (8, 5)

total_test = pd.read_csv('/content/drive/Shared drives/빅콘테스트2020/total_test.csv')

# 테스트 데이터에서 2020년 6월 1일의 row는 06시부터 02시(다음날이지만 한 싸이클로 포함)까지 79개
june_1st = total_test.iloc[:79, :]

# 표시된 것 이외에는 1시간에 3개 row씩 존재
product1 = june_1st.iloc[:3,:]
product2 = june_1st.iloc[3:6,:]
product3 = june_1st.iloc[6:9,:]
product4 = june_1st.iloc[9:12,:]
product5 = june_1st.iloc[12:15,:]
product6 = june_1st.iloc[15:18,:]
product7 = june_1st.iloc[18:21,:]
product8 = june_1st.iloc[21:27,:] # 6개
product9 = june_1st.iloc[27:28,:] # 1개
product10 = june_1st.iloc[28:31,:]
product11 = june_1st.iloc[31:34,:]
product12 = june_1st.iloc[34:40,:] # 6개
product13 = june_1st.iloc[40:43,:]
product14 = june_1st.iloc[43:46,:]
product15 = june_1st.iloc[46:49,:]
product16 = june_1st.iloc[49:58,:] # 9개
product17 = june_1st.iloc[58:67,:] # 9개
product18 = june_1st.iloc[67:73,:] # 6개
product19 = june_1st.iloc[73:76,:]
product20 = june_1st.iloc[76:79,:]

# 각 시간대별 예측을 위해 시간 변수를 바꿔서 늘린 테스트 데이터의 마더코드별 데이터셋 구축
product1_total = pd.DataFrame()
product2_total = pd.DataFrame()
product3_total = pd.DataFrame()
product4_total = pd.DataFrame()
product5_total = pd.DataFrame()
product6_total = pd.DataFrame()
product7_total = pd.DataFrame()
product8_total = pd.DataFrame()
product9_total = pd.DataFrame()
product10_total = pd.DataFrame()
product11_total = pd.DataFrame()
product12_total = pd.DataFrame()
product13_total = pd.DataFrame()
product14_total = pd.DataFrame()
product15_total = pd.DataFrame()
product16_total = pd.DataFrame()
product17_total = pd.DataFrame()
product18_total = pd.DataFrame()
product19_total = pd.DataFrame()
product20_total = pd.DataFrame()

# product1
for i in range(20):
  temp = product1.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product1_total = product1_total.append(temp)
  product1_total = product1_total.reset_index(drop=True)

# product2
for i in range(20):
  temp = product2.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product2_total = product2_total.append(temp)
  product2_total = product2_total.reset_index(drop=True)

# product3
for i in range(20):
  temp = product3.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product3_total = product3_total.append(temp)
  product3_total = product3_total.reset_index(drop=True)

# product4
for i in range(20):
  temp = product4.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product4_total = product4_total.append(temp)
  product4_total = product4_total.reset_index(drop=True)

# product5
for i in range(20):
  temp = product5.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product5_total = product5_total.append(temp)
  product5_total = product5_total.reset_index(drop=True)

# product6
for i in range(20):
  temp = product6.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product6_total = product6_total.append(temp)
  product6_total = product6_total.reset_index(drop=True)

# product7
for i in range(20):
  temp = product7.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product7_total = product7_total.append(temp)
  product7_total = product7_total.reset_index(drop=True)

# product8
for i in range(20):
  temp = product8.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+7)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 6
  product8_total = product8_total.append(temp)
  product8_total = product8_total.reset_index(drop=True)

# product9
for i in range(20):
  temp = product9.copy()
  temp['시'] = [(i+6)%24]
  temp['시간대'] = [str((i + 6)%24) + '시']
  product9_total = product9_total.append(temp)
  product9_total = product9_total.reset_index(drop=True)

# product10
for i in range(20):
  temp = product10.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product10_total = product10_total.append(temp)
  product10_total = product10_total.reset_index(drop=True)

# product11
for i in range(20):
  temp = product11.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product11_total = product11_total.append(temp)
  product11_total = product11_total.reset_index(drop=True)

# product12
for i in range(20):
  temp = product12.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+7)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 6
  product12_total = product12_total.append(temp)
  product12_total = product12_total.reset_index(drop=True)

# product13
for i in range(20):
  temp = product13.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product13_total = product13_total.append(temp)
  product13_total = product13_total.reset_index(drop=True)

# product14
for i in range(20):
  temp = product14.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product14_total = product14_total.append(temp)
  product14_total = product14_total.reset_index(drop=True)

# product15
for i in range(20):
  temp = product15.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product15_total = product15_total.append(temp)
  product15_total = product15_total.reset_index(drop=True)

# product16
for i in range(20):
  temp = product16.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+7)%24, (i+7)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 9
  product16_total = product16_total.append(temp)
  product16_total = product16_total.reset_index(drop=True)

# product17
for i in range(20):
  temp = product17.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+7)%24, (i+7)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 9
  product17_total = product17_total.append(temp)
  product17_total = product17_total.reset_index(drop=True)

# product18
for i in range(20):
  temp = product18.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+6)%24, (i+6)%24, (i+7)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 6
  product18_total = product18_total.append(temp)
  product18_total = product18_total.reset_index(drop=True)

# product19
for i in range(20):
  temp = product19.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product19_total = product19_total.append(temp)
  product19_total = product19_total.reset_index(drop=True)

# product20
for i in range(20):
  temp = product20.copy()
  temp['시'] = [(i+6)%24, (i+6)%24, (i+7)%24]
  temp['시간대'] = [str((i + 6)%24) + '시'] * 3
  product20_total = product20_total.append(temp)
  product20_total = product20_total.reset_index(drop=True)

augmented_test = pd.concat([product1_total, product2_total, product3_total, product4_total, product5_total,
                            product6_total, product7_total, product8_total, product9_total, product10_total,
                            product11_total, product12_total, product13_total, product14_total, product15_total,
                            product16_total, product17_total, product18_total, product19_total, product20_total], axis=0)

# 사용할 변수 리스트
col = ['emb_label_주문량', 'emb_label2_주문량', 'emb_label3_주문량', '월_주문량', '요일_주문량', '시등급_주문량', '분등급_주문량','노출횟수_주문량', '판매단가',
       'emb_label','emb_label2','emb_label3', '시', '분' ,'월', '요일','노출횟수']

# 사용할 변수 리스트에 임베딩 컬럼 추가
col.extend([str(x) for x in range(159)])
len(col)

augmented_test_fit = augmented_test[col]

scaler = RobustScaler()
scaler.fit(augmented_test_fit)
augmented_test_final = pd.DataFrame(scaler.transform(augmented_test_fit), columns=augmented_test_fit.columns)

def build_model(input_layer, start_neurons):
    
    conv1 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same", kernel_initializer='he_normal', input_shape=(augmented_test_final.shape[1], 1))(input_layer)
    conv1 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same", kernel_initializer='he_normal')(conv1)
    pool1 = BatchNormalization()(conv1)
    pool1 = MaxPooling1D(2)(pool1)
    pool1 = Dropout(0.1)(pool1)

    conv2 = Conv1D(start_neurons * 2, 3, activation="elu", padding="same", kernel_initializer='he_normal')(pool1)
    conv2 = Conv1D(start_neurons * 2, 3, activation="elu", padding="same", kernel_initializer='he_normal')(conv2)
    pool2 = BatchNormalization()(conv2)
    pool2 = MaxPooling1D(2)(pool2)
    pool2 = Dropout(0.1)(pool2)

    conv3 = Conv1D(start_neurons * 2, 3, activation="elu", padding="same", kernel_initializer='he_normal')(pool2)
    conv3 = Conv1D(start_neurons * 2, 3, activation="elu", padding="same", kernel_initializer='he_normal')(conv3)
    pool3 = BatchNormalization()(conv3)
    pool3 = AveragePooling1D(2)(pool3)
    pool3 = Dropout(0.1)(pool3)


    convm = Conv1D(start_neurons * 4, 3, activation="elu", padding="same",kernel_initializer='he_normal')(pool3)

    deconv3 = Conv1DTranspose(start_neurons * 1, 3, strides=2, padding="same",kernel_initializer='he_normal')(convm)
    uconv3 = concatenate([deconv3, conv3])
    uconv3 = Dropout(0.1)(uconv3)
    uconv3 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same",kernel_initializer='he_normal')(uconv3)
    uconv3 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same",kernel_initializer='he_normal')(uconv3)
    uconv3 = BatchNormalization()(uconv3)
    uconv3 = Dropout(0.1)(uconv3)

    deconv2 = Conv1DTranspose(start_neurons * 1, 3, strides=2, padding="same",kernel_initializer='he_normal')(uconv3)
    uconv2 = concatenate([deconv2, conv2])
    uconv2 = Dropout(0.1)(uconv2)
    uconv2 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same",kernel_initializer='he_normal')(uconv2)
    uconv2 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same",kernel_initializer='he_normal')(uconv2)
    uconv2 = BatchNormalization()(uconv2)
    uconv2 = Dropout(0.1)(uconv2)

    deconv1 = Conv1DTranspose(start_neurons * 1, 3, strides=2, padding="same",kernel_initializer='he_normal')(uconv2)
    uconv1 = concatenate([deconv1, conv1])
    uconv1 = Dropout(0.1)(uconv1)
    uconv1 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same",kernel_initializer='he_normal')(uconv1)
    uconv1 = Conv1D(start_neurons * 1, 3, activation="elu", padding="same",kernel_initializer='he_normal')(uconv1)
    uconv1 = BatchNormalization()(uconv1)
    uconv1 = Dropout(0.1)(uconv1)

    flat = Flatten()(uconv1)
    dense1 = Dense(256, activation='elu')(flat)
    dense2 = Dense(128, activation='elu')(dense1)
    output_layer = Dense(1)(dense2)
    
    return output_layer

# model 생성
input_layer = Input((augmented_test_final.shape[1], 1))
output_layer = build_model(input_layer, 128)
model = Model(input_layer, output_layer)

# model 가중치 불러오기
model.load_weights('/content/drive/Shared drives/빅콘테스트2020/weights/ds/functional_3.44-35.0913.hdf5')

# 수량 예측
y_pred = model.predict(augmented_test_final.values)

y_pred = y_pred.reshape(-1)

augmented_test['취급액'] = y_pred * augmented_test['판매단가']

# 2020년 6월 1일 각 상품의 시간대별 예상 취급액
mat = pd.pivot_table(augmented_test, index='마더코드', values='취급액', aggfunc='sum', columns='시간대')

sorted_cols = ['6시', '7시', '8시', '9시', '10시', '11시', '12시', '13시', '14시', '15시',
               '16시', '17시', '18시', '19시', '20시', '21시', '22시', '23시', '0시', '1시']

mat = mat[sorted_cols]

mat

# maxtrix 내 선택을 위한 빈 slot 데이터프레임 만들기
slot = mat.copy()
slot.iloc[:, :] = 0

# 두 개의 취급액의 한 시간대를 놓고 경합할 경우의 확률적 선택 계산
def probability_matching(column, prod1, prod2):
    sale1 = mat.iloc[prod1, column] + 1e-15
    sale2 = mat.iloc[prod2, column] + 1e-15
    prob_sale1 = round(sale1 / (sale1 + sale2), 2)
    prob_sale2 = round(sale2 / (sale1 + sale2), 2)
    package = [prod1, prod2]
    return random.choices(package, weights=[prob_sale1, prob_sale2])[0]

# MSPMA 알고리즘
def mspma(now_row, max_col):
    # 현재 상태의 값을 slot에 표시
    slot.iloc[now_row, max_col] = 1
    
    # slot 테이블에서 max_col에 로우 두 개가 1로 된 경우
    temp = np.array(slot.iloc[:, max_col])
    if len(temp[temp == 1]) == 2:
        
        # now_row외에 다른 1의 row위치를 찾기
        temp_col = list(slot.iloc[:, max_col])
        temp_col[now_row] = -1
        compare_row = np.where(np.array(temp_col) == 1)[0][0]
        
        # 확률 매칭 함수 적용
        result = probability_matching(max_col, now_row, compare_row)
        
        # 비교 대상(compare_row)이 선택되는 경우 (now_row의 max_col 위치가 바뀌어야 하는 상황)
        if result == compare_row:
            # now_row기존의 값을 -1로 대체
            slot.iloc[now_row, max_col] = -1
            mat.iloc[now_row, max_col] = -1
            
            # now_row에 0만 있는 것들 중 가장 큰 것의 인덱스 찾기
            max_col = np.argmax(mat.iloc[now_row, :])

            # slot에 현재 now_row에서 가장 큰 값이라고 표시
            slot.iloc[now_row, max_col] = 1
            
            # 재귀 함수
            mspma(now_row, max_col)

        # 현재 상품(now_row)이 선택되는 경우 (compare_row의 max_col 위치가 바뀌어야 하는 상황)
        else:
            # compare_row기존의 값을 -1로 대체
            slot.iloc[compare_row, max_col] = -1
            mat.iloc[compare_row, max_col] = -1
            
            # compare_row에 0만 있는 것들 중 가장 큰 것의 인덱스 찾기
            max_col = np.argmax(mat.iloc[now_row, :])
            
            # slot에 현재 compare_row에서 가장 큰 값이라고 표시
            slot.iloc[compare_row, max_col] = 1
            
            # 재귀함수
            mspma(compare_row, max_col)

# 최종 구현
start_row = 0

for i in range(20):
    new_col = np.argmax(mat.iloc[start_row, :])   
    mspma(start_row, new_col)
    start_row += 1

def isHangul(text):
    pyVer3 =  sys.version_info >= (3, 0)

    if pyVer3 : # for Ver 3 or later
        encText = text
    else: # for Ver 2.x
        if type(text) is not unicode:
            encText = text.decode('utf-8')
        else:
            encText = text

    hanCount = len(re.findall(u'[\u3130-\u318F\uAC00-\uD7A3]+', encText))
    return hanCount > 0

for i in slot.columns:
    mother_code = slot[i][slot[i] == 1].index[0]
    name = june_1st[june_1st['마더코드'] == mother_code]['상품명']
    
    cnt = 0
    temp = set(name.values)
    for j in temp:
      for k in range(len(j)):
            temp2 = j[k]
            if isHangul(temp2):
                cnt += 2
            else:
                cnt += 1
            
    print('*')
    if len(i) > 2:
        print('--------  ', '------------------', ' '+'-'*cnt+'--'+'-'*(4*len(temp)-1)+'-------')
        print('|', i+' |','-> 마더코드:', mother_code,'| 상품:', set(name.values),'|')
        print('--------  ', '------------------', ' '+'-'*cnt+'--'+'-'*(4*len(temp)-1)+'-------')
    else:
        print('-------  ', '------------------', ' '+'-'*cnt+'--'+'-'*(4*len(temp)-1)+'------')
        print('|', i+' |','-> 마더코드:', mother_code,'| 상품:', str(temp)+'|')
        print('-------  ', '------------------', ' '+'-'*cnt+'--'+'-'*(4*len(temp)-1)+'------')

sclr = RobustScaler()
sclr.fit(june_1st[col])
test_scaled = pd.DataFrame(sclr.transform(june_1st[col]), columns=june_1st[col].columns)

# model 생성
input_layer = Input((test_scaled.shape[1], 1))
output_layer = build_model(input_layer, 128)
model = Model(input_layer, output_layer)

# model 가중치 불러오기
model.load_weights('/content/drive/Shared drives/빅콘테스트2020/weights/ds/functional_3.44-35.0913.hdf5')

# 수량 예측
y_pred = model.predict(test_scaled.values)

y_pred = y_pred.reshape(-1)

sum(june_1st['판매단가'] * y_pred)

new_sum = 0

for i in slot.columns:
    mother_code = slot[i][slot[i] == 1].index[0]
    new_sum += mat[i][mother_code]

new_sum

df = pd.DataFrame({'before': [sum(june_1st['판매단가'] * y_pred)], 'after':[new_sum]}).T
df.columns= ['취급액']
df.reset_index(inplace=True)

sns.barplot(x="index", y="취급액", data=df)