Skip to content

TensorFlow for training with multiple api added allowing early/late fusion #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sadekr wants to merge 1 commit into
base: master
Choose a base branch
from
Open

TensorFlow for training with multiple api added allowing early/late fusion #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 149 additions & 27 deletions ML/python_training.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,13 @@
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers.experimental import preprocessing
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.models import Model
from tensorflow.keras.utils import plot_model
from tensorflow.keras.layers import concatenate
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import LayerNormalization

import keras2onnx
import tf2onnx
Expand Down Expand Up @@ -81,6 +86,7 @@ def get_external_params():
global SETUP_PARAMS_KEY
global OUTPUT_NAME_KEY
global DEBUG_MODE_KEY
global STRUCTURE_KEY

parser = argparse.ArgumentParser()
parser.add_argument("config")
Expand All @@ -92,6 +98,7 @@ def get_external_params():
MODEL_KEY = obj['MODEL']
INPUT_NAME_KEY = obj['INPUT_NAME']
LOAD_PARAMS_KEY = obj['LOAD_PARAMS']
STRUCTURE_KEY = obj['STRUCTURE']
CALC_PARAMS_KEY = obj['CALC_PARAMS']
OBJECT_KEY = obj['OBJECT']
HYPER_PARAM_KEY = obj['HYPER_PARAM']
Expand Down Expand Up @@ -180,32 +187,146 @@ def create_model_TensorFlowNN():
import pickle
global model

normalize = preprocessing.Normalization()

model = tf.keras.Sequential([normalize,
layers.Dense(x_train.shape[1], use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(x_train.shape[1], activation='sigmoid',use_bias=False),
layers.Dense(1, activation='sigmoid')])

model.compile(loss='binary_crossentropy', optimizer=tf.optimizers.Adam(learning_rate=0.005), metrics=['accuracy'])

training_history = model.fit(x_train, y_train,
epochs=1500,
batch_size=1024,
verbose=1,
validation_data=(x_valid, y_valid))
METRICS = [
keras.metrics.BinaryAccuracy(name='accuracy'),
keras.metrics.Precision(name='precision'),
keras.metrics.Recall(name='recall'),
keras.metrics.AUC(name='auc'),
keras.metrics.AUC(name='prc', curve='PR')]

modelA=STRUCTURE_KEY['modelA']
modelB=STRUCTURE_KEY['modelB']
modelM=STRUCTURE_KEY['modelM']

modelA=np.array(modelA)
modelB=np.array(modelB)
modelM=np.array(modelM)

if (modelM.shape[0] != 0):
MultipleInput = 1
else:
MultipleInput = 0

if (MultipleInput == 0):
modelA_inputShape = int(modelA[0]);
modelA_Normalization = int(modelA[1]);
A1 = Input(shape = (modelA_inputShape,))
lst = np.size(modelA);
it= lst/3;
if (modelA_Normalization == 1):
A = LayerNormalization(axis=-1)(A1)
A = Dense(int(modelA[3]), use_bias=False)(A)
else:
A = Dense(int(modelA[3]), use_bias=False)(A1)
modelA_Shape = modelA[3::3]
modelA_Normalization= modelA[4::3]
modelA_Dropout = modelA[5::3]
for i in range(0,int(it)-1):
if ((i != 0) or (int(modelA_Shape[i]) != 0)):
A = Dense(int(modelA_Shape[i]), activation='sigmoid',use_bias=False)(A)
if (int(modelA_Normalization[i]) == 1):
A = LayerNormalization(axis=-1)(A)
if (modelA_Dropout[i] != 0.0):
A = Dropout(modelA_Dropout[i])(A)
A = Model(inputs=A1, outputs=A)

#print(A.summary())
#plot_model(A,to_file="ModelStructA.png",show_shapes=True)

A.compile(loss='binary_crossentropy', optimizer=tf.optimizers.Adam(learning_rate=HYPER_PARAM_KEY['LEARN_RATE']), metrics=METRICS)
training_history = A.fit(x_train, y_train,
epochs=HYPER_PARAM_KEY['EPOCHS'],
batch_size=HYPER_PARAM_KEY['BATCH_SIZE'],
verbose=HYPER_PARAM_KEY['VERBOSE'],
validation_split=HYPER_PARAM_KEY['VALIDATION_SPLIT'])

with open(OUTPUT_NAME_KEY['ORIGIN'],"wb") as f:
A.save('tf_model')

else:
modelA_inputShape = int(modelA[0]);
modelA_Normalization = int(modelA[1]);
A1 = Input(shape = (modelA_inputShape,))
lst = np.size(modelA);
it= lst/3;
if (modelA_Normalization == 1):
A = LayerNormalization(axis=-1)(A1)
A = Dense(int(modelA[3]), use_bias=False)(A)
else:
A = Dense(int(modelA[3]), use_bias=False)(A1)
modelA_Shape = modelA[3::3]
modelA_Normalization= modelA[4::3]
modelA_Dropout = modelA[5::3]

for i in range(0,int(it)-1):
if ((i != 0) or (int(modelA_Shape[i]) != 0)):
A = Dense(int(modelA_Shape[i]), activation='sigmoid',use_bias=False)(A)
if (int(modelA_Normalization[i]) == 1):
A = LayerNormalization(axis=-1)(A)
if (modelA_Dropout[i] != 0.0):
A = Dropout(modelA_Dropout[i])(A)
A = Model(inputs=A1, outputs=A)

modelB_inputShape = int(modelB[0]);
modelB_Normalization = int(modelB[1]);
B1 = Input(shape = (modelB_inputShape,))
lst = np.size(modelB);
it= lst/3;
if (modelB_Normalization == 1):
B = LayerNormalization(axis=-1)(B1)
B = Dense(int(modelB[3]), use_bias=False)(B)
else:
B = Dense(int(modelB[3]), use_bias=False)(B1)

modelB_Shape = modelB[3::3]
modelB_Normalization= modelB[4::3]
modelB_Dropout = modelB[5::3]
for i in range(0,int(it)-1):
if ((i != 0) or (int(modelB_Shape[i]) != 0)):
B = Dense(int(modelB_Shape[i]), activation='sigmoid',use_bias=False)(B)
if (int(modelB_Normalization[i]) == 1):
B = LayerNormalization(axis=-1)(B)
if (modelB_Dropout[i] != 0.0):
B = Dropout(modelB_Dropout[i])(B)
B = Model(inputs=B1, outputs=B)

M1 = concatenate([A.output,B.output])
modelM_Normalization = int(modelM[1]);
lst = np.size(modelM);
it= lst/3;
if (modelM_Normalization == 1):
M = LayerNormalization(axis=-1)(M1)
M = Dense(int(modelM[3]), use_bias=False)(M)
else:
M = Dense(int(modelM[3]), use_bias=False)(M1)
modelM_Shape = modelM[3::3]
modelM_Normalization= modelM[4::3]
modelM_Dropout = modelM[5::3]
for i in range(0,int(it)-1):
if ((i != 0) or (int(modelM_Shape[i]) != 0)):
M = Dense(int(modelM_Shape[i]), activation='sigmoid',use_bias=False)(M)
if (int(modelM_Normalization[i]) == 1):
M = LayerNormalization(axis=-1)(M)
if (modelM_Dropout[i] != 0.0):
M = Dropout(modelM_Dropout[i])(M)
M = Dense(1, activation='sigmoid',use_bias=False)(M)
full_mod = Model(inputs=[A.input, B.input], outputs=M)

#print(full_mod.summary())
#plot_model(full_mod,to_file="ModelStruct.png",show_shapes=True)

x1_train, x2_train = np.hsplit(x_train, [int(modelA[0])]);

full_mod.compile(loss='binary_crossentropy', optimizer=tf.optimizers.Adam(learning_rate=HYPER_PARAM_KEY['LEARN_RATE']), metrics=METRICS)
training_history = full_mod.fit(x=[x1_train,x2_train], y=y_train,
epochs=HYPER_PARAM_KEY['EPOCHS'],
batch_size=HYPER_PARAM_KEY['BATCH_SIZE'],
verbose=HYPER_PARAM_KEY['VERBOSE'],
validation_split=HYPER_PARAM_KEY['VALIDATION_SPLIT'])

with open(OUTPUT_NAME_KEY['ORIGIN'],"wb") as f:
full_mod.save('tf_model')

with open(OUTPUT_NAME_KEY['ORIGIN'],"wb") as f:
model.save('tf_model')
#pickle.dump(model,f)

def convert_onnx_TensorFlowNN():

Expand Down Expand Up @@ -461,6 +582,7 @@ def main():
global SETUP_PARAMS_KEY
global OUTPUT_NAME_KEY
global DEBUG_MODE_KEY
global STRUCTURE_KEY

global df
global df_train
Expand Down
25 changes: 17 additions & 8 deletions ML/training_configure_tfNN.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,20 @@ SETUP_PARAMS:
HYPER_PARAM:
TYPE: gbdt
OBJECTIVE: binary
LEARN_RATE: 0.01
MAX_DEPTH: 5
N_ESTIMATOR: 1000
MAX_BIN: 20000
LEARN_RATE: 0.0005
METRIC: auc
EPOCHS: 500
BATCH_SIZE: 32
VERBOSE: 1
VALIDATION_SPLIT: 0.2

STRUCTURE:
modelA:
[]
modelB:
[]
modelM:
[]

LOAD_PARAMS:
- MFT_X
Expand Down Expand Up @@ -61,14 +70,14 @@ CALC_PARAMS:

TRAIN_PARAMS:
- MFT_X
- MFT_Y
- MFT_Phi
- MFT_Tanl
- MFT_InvQPt
- MCH_X
- MFT_Y
- MCH_Y
- MFT_Phi
- MCH_Phi
- MFT_Tanl
- MCH_Tanl
- MFT_InvQPt
- MCH_InvQPt
- MFT_TrackChi2
- Delta_X
Expand Down