Notes

from com.highradius.ml_automation.model_trainers.ModelTrainerTemplate import Modeltrainer
from com.highradius.Products.Deductions.valid_invalid.feature_functions.BValueCalculation import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.OneLevelHistory import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.OriginalByAverageDisputeAmount import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.DateDifference import *

from com.highradius.Products.Deductions.valid_invalid.feature_functions.isPresent import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.MapToValue import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.MonthOfDate import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.RowLevelHistory import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.TwoLevelHistory import *
from com.highradius.Products.Deductions.valid_invalid.feature_functions.MapToValue import *

from sklearn.externals import joblib
from sklearn.ensemble import RandomForestClassifier
from sklearn2pmml import PMMLPipeline
from sklearn.pipeline import Pipeline
from sklearn_pandas import DataFrameMapper
from sklearn2pmml import sklearn2pmml
import pandas as pd
import lightgbm as lgb
from sklearn import datasets
from sklearn import tree
import warnings

warnings.filterwarnings('ignore')


class DuracellModelTrainer(Modeltrainer):
def performModelTraining(self, preprocessed_df, aggregation_dfs, model_target_file_path, model_arguments,
modelGeneratorContext):

#check
date_column = model_arguments[0]
training_start_date = pd.to_datetime(model_arguments[1])
training_end_date = pd.to_datetime(model_arguments[2])
preprocessed_df[date_column] = pd.to_datetime(preprocessed_df[date_column])
train_data = preprocessed_df.loc[
(preprocessed_df[date_column] >= training_start_date) & (preprocessed_df[date_column] <= training_end_date)]
train_data.reset_index(inplace=True, drop=True)
# left_history = pd.DataFrame(columns=train_data.columns)
empty_df = pd.DataFrame(columns=train_data.columns.tolist())


train_data['ar_reason_code_history'] = one_level_history(train_data.copy(), level_1='ar_reason_code',
aggregated_df=aggregation_dfs[3],
column=['ar_reason_code_history'], left_history=empty_df,
output_label='isValid',
fold=5, regularized=True)

train_data['row_history'] = row_level_hstry(train_data.copy().copy(), 'fk_customer_map_id',
'original_dispute_amount',
left_history=empty_df, output_label='isValid',
aggregated_df1=aggregation_dfs[0],
aggregated_df2=aggregation_dfs[4],
column=['customer_avg_amount', 'row_level'],
fold=5,
regularized=True)

temp = train_data.copy()
temp['deduction_created_date'] = pd.to_datetime(temp['deduction_created_date'])
train_data['deduction_created_date_quarter'] = 2 * temp['deduction_created_date'].dt.quarter

train_data['payer_invalid_history'] = one_level_history(train_data.copy(), level_1='fk_customer_map_id',
aggregated_df=aggregation_dfs[0],
column=['payer_invalid_history'],
left_history = empty_df,
output_label='isValid',
fold=5,
regularized=True)

train_data['company_code_history'] = one_level_history(train_data.copy(),
level_1='company_code',
aggregated_df=aggregation_dfs[1],
column=['company_code_history'],
left_history=empty_df, output_label='isValid',
fold=5,
regularized=True)


train_data['customer_ar_reason_code_history'] = two_level_hstry(train_data.copy(),
'fk_customer_map_id,ar_reason_code',
left_history=empty_df,
aggregated_df=aggregation_dfs[2],
column=['customer_ar_reason_code_history'])


# Features
training_features = train_data[[
'ar_reason_code',
'deduction_created_date_quarter',
'customer_ar_reason_code_history',
'company_code_history',
'payer_invalid_history',
'ar_reason_code_history',
'row_history',
'original_dispute_amount'
]]

training_label = train_data.loc[:, 'isValid']

model = lgb.LGBMClassifier(class_weight= {0: 146.58961187214612, 1: 0.5017112784187475}, max_depth=4,
bagging_fraction = 0.4,
min_child_samples=197,
min_child_weight=100,
num_leaves=32,
reg_alpha=0.1,
reg_lambda=50,
subsample=0.342279949797131,
colsample_bytree=0.6764980114339743,
verbose = -1
)

mapper = DataFrameMapper([('ar_reason_code', None),
('deduction_created_date_quarter', None),
('customer_ar_reason_code_history', None),
('company_code_history', None),
('payer_invalid_history', None),
('ar_reason_code_history', None),
('row_history', None),
('original_dispute_amount', None)
])

# generating pmml
pmmlpipeline = PMMLPipeline([("mapper", mapper), ("estimator", model)])
pmmlpipeline.fit(training_features, training_label)
sklearn2pmml(pmmlpipeline, model_target_file_path, user_classpath=[r"sklearn2pmml-plugin-1.0-SNAPSHOT.jar"],
debug=False)

# generating pickle for python pipeline
pythonpipeline = Pipeline([('mapper', mapper), ('model', model)])
pythonpipeline.fit(training_features, training_label)
pickle_path = model_target_file_path.split(".")
pickle_path = pickle_path[0] + ".pkl"
joblib.dump(pythonpipeline, pickle_path)

return "Success"