### Data extraction
import numpy as np
import math
import time
import h5py
import numpy as np
import os
import sys
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import pickle
import sklearn
import pickle
from numpy import array
from keras.layers import Permute
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import Input
from tensorflow.keras.models import Model
from sklearn.model_selection import train_test_split
from tensorflow.keras import optimizers
from tensorflow import losses

physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
tf.config.experimental.set_memory_growth(physical_devices[0], True)




print(tf.__version__)
print(sys.version_info)
for module in mpl,np,pd,sklearn,tf,keras :
     print(module.__name__,module.__version__)
folder = '/waveform_example'
folder

###Data preparation script
dataset_y = []
dataset_x = []
for root,dirs,files in os.walk(folder):
    #scorri i file dentro la cartella
    for filename in files:
    #controlla che l'estensione del file sia .hdf5
     if filename.endswith((".hdf5")):
        print(filename)
        #create_dir(folder+'/'+filename)
        #naviga i deversi livelli del file
        f = h5py.File(folder+'/'+filename,'r')
        liv_1 = list(f.keys())
        for I in liv_1:
            data = f[I]
            liv_2 = list(data.keys())
            for ch in liv_2:
            #Considera solo i dati che al livello 2 hanno CH_A
                    if ch == 'CH_A':
                       print(ch)
                       data[ch].keys()
                       liv_3 = list(data[ch].keys())
                       for trg in liv_3:
                    #print(trg)
                    #Estrai la serie storica
                          if max(data[ch][trg][()])<75000:
                             x = data[ch][trg][()]
                           #Agguingi come ultimo elemento il valore della variabile
                           #target per la serie storica sulla base del nome del file di partenza
                          if 'Am' in filename:
                             dataset_y.append(1)
                          else:
                             dataset_y.append(0)
                    #Appendi alla lista "dataset" il valore della serei storica con relativa etichetta
                          dataset_x.append(x)
    #Salvare ne lformato che si ritiene piu comodo
X = dataset_x.copy()
print(X)

X = [-i for i in X]
print(X)
y = dataset_y.copy()

np.random.seed(12)
np.random.shuffle(X)
np.random.seed(12)
np.random.shuffle(y)


x_train,x_test,y_train,y_test = train_test_split(X,y,test_size = 0.1)
x_train,x_valid,y_train,y_valid = train_test_split(x_train,y_train,test_size = 0.2)

x_train = np.array(x_train)
y_train = np.array(y_train)
x_valid = np.array(x_valid)
y_valid = np.array(y_valid)
x_test = np.array(x_test)
y_test = np.array(y_test)



x_train= np.reshape(x_train,(x_train.shape[0],x_train.shape[1],1))
x_valid= np.reshape(x_valid,(x_valid.shape[0],x_valid.shape[1],1))
x_test = np.reshape(x_test,(x_test.shape[0],x_test.shape[1],1))



print(x_train.shape,
y_train.shape,
x_valid.shape,
y_valid.shape,
x_test.shape,
y_test.shape)

t0 = time.time()

input_fca = Input(shape=(16384,1))





l1 = tf.keras.layers.GRU(units = 32,dropout= 0.5,return_sequences=True)(input_fca)
print(l1)

l2 = keras.layers.Bidirectional(keras.layers.GRU(units=64,dropout = 0.7,return_sequences=False))(l1)

l3 = tf.keras.layers.Dense(64,activation= 'selu')(l2)

l4 = tf.keras.layers.Dense(32,activation= 'selu')(l3)

output__layer = tf.keras.layers.Dense(1,activation= 'sigmoid')(l4)



model = Model(inputs=[input_fca], outputs=[output__layer])

model.summary()

initia_learning_rate = 0.01

lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initia_learning_rate,
    decay_steps=899,
    decay_rate=0.1,
    staircase=False)
model.compile(optimizer= keras.optimizers.Adam(learning_rate=lr_schedule),loss = tf.losses.BinaryCrossentropy(), metrics= ['accuracy'])

history = model.fit(x_train, y_train,validation_data= ([x_valid,y_valid]),epochs=10,batch_size= 64)
def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize = (8,5))
    plt.grid(True)
    plt.gca().set_ylim(0,2)
    plt.savefig("/xxx.png")
    plt.show()

score_train = model.evaluate(x_train,y_train)
score_test = model.evaluate(x_test,y_test)
print(score_train,score_test)


t1 = time.time()
print('total time cost:',t1-t0)

plot_learning_curves(history)




model.save('/xxx.h5')
ac7118bafeb14545982665270582c02427bf4ccd
<mask token>


def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize=(8, 5))
    plt.grid(True)
    plt.gca().set_ylim(0, 2)
    plt.savefig('/xxx.png')
    plt.show()


<mask token>

<mask token>
assert len(physical_devices) > 0, 'Not enough GPU hardware devices available'
tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(tf.__version__)
print(sys.version_info)
for module in (mpl, np, pd, sklearn, tf, keras):
    print(module.__name__, module.__version__)
<mask token>
folder
<mask token>
for root, dirs, files in os.walk(folder):
    for filename in files:
        if filename.endswith('.hdf5'):
            print(filename)
            f = h5py.File(folder + '/' + filename, 'r')
            liv_1 = list(f.keys())
            for I in liv_1:
                data = f[I]
                liv_2 = list(data.keys())
                for ch in liv_2:
                    if ch == 'CH_A':
                        print(ch)
                        data[ch].keys()
                        liv_3 = list(data[ch].keys())
                        for trg in liv_3:
                            if max(data[ch][trg][()]) < 75000:
                                x = data[ch][trg][()]
                            if 'Am' in filename:
                                dataset_y.append(1)
                            else:
                                dataset_y.append(0)
                            dataset_x.append(x)
<mask token>
print(X)
<mask token>
print(X)
<mask token>
np.random.seed(12)
np.random.shuffle(X)
np.random.seed(12)
np.random.shuffle(y)
<mask token>
print(x_train.shape, y_train.shape, x_valid.shape, y_valid.shape, x_test.
    shape, y_test.shape)
<mask token>
print(l1)
<mask token>
model.summary()
<mask token>
model.compile(optimizer=keras.optimizers.Adam(learning_rate=lr_schedule),
    loss=tf.losses.BinaryCrossentropy(), metrics=['accuracy'])
<mask token>


def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize=(8, 5))
    plt.grid(True)
    plt.gca().set_ylim(0, 2)
    plt.savefig('/xxx.png')
    plt.show()


<mask token>
print(score_train, score_test)
<mask token>
print('total time cost:', t1 - t0)
plot_learning_curves(history)
model.save('/xxx.h5')

<mask token>
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, 'Not enough GPU hardware devices available'
tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(tf.__version__)
print(sys.version_info)
for module in (mpl, np, pd, sklearn, tf, keras):
    print(module.__name__, module.__version__)
folder = '/waveform_example'
folder
dataset_y = []
dataset_x = []
for root, dirs, files in os.walk(folder):
    for filename in files:
        if filename.endswith('.hdf5'):
            print(filename)
            f = h5py.File(folder + '/' + filename, 'r')
            liv_1 = list(f.keys())
            for I in liv_1:
                data = f[I]
                liv_2 = list(data.keys())
                for ch in liv_2:
                    if ch == 'CH_A':
                        print(ch)
                        data[ch].keys()
                        liv_3 = list(data[ch].keys())
                        for trg in liv_3:
                            if max(data[ch][trg][()]) < 75000:
                                x = data[ch][trg][()]
                            if 'Am' in filename:
                                dataset_y.append(1)
                            else:
                                dataset_y.append(0)
                            dataset_x.append(x)
X = dataset_x.copy()
print(X)
X = [(-i) for i in X]
print(X)
y = dataset_y.copy()
np.random.seed(12)
np.random.shuffle(X)
np.random.seed(12)
np.random.shuffle(y)
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train,
    test_size=0.2)
x_train = np.array(x_train)
y_train = np.array(y_train)
x_valid = np.array(x_valid)
y_valid = np.array(y_valid)
x_test = np.array(x_test)
y_test = np.array(y_test)
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
x_valid = np.reshape(x_valid, (x_valid.shape[0], x_valid.shape[1], 1))
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
print(x_train.shape, y_train.shape, x_valid.shape, y_valid.shape, x_test.
    shape, y_test.shape)
t0 = time.time()
input_fca = Input(shape=(16384, 1))
l1 = tf.keras.layers.GRU(units=32, dropout=0.5, return_sequences=True)(
    input_fca)
print(l1)
l2 = keras.layers.Bidirectional(keras.layers.GRU(units=64, dropout=0.7,
    return_sequences=False))(l1)
l3 = tf.keras.layers.Dense(64, activation='selu')(l2)
l4 = tf.keras.layers.Dense(32, activation='selu')(l3)
output__layer = tf.keras.layers.Dense(1, activation='sigmoid')(l4)
model = Model(inputs=[input_fca], outputs=[output__layer])
model.summary()
initia_learning_rate = 0.01
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initia_learning_rate, decay_steps=899, decay_rate=0.1, staircase=False)
model.compile(optimizer=keras.optimizers.Adam(learning_rate=lr_schedule),
    loss=tf.losses.BinaryCrossentropy(), metrics=['accuracy'])
history = model.fit(x_train, y_train, validation_data=[x_valid, y_valid],
    epochs=10, batch_size=64)


def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize=(8, 5))
    plt.grid(True)
    plt.gca().set_ylim(0, 2)
    plt.savefig('/xxx.png')
    plt.show()


score_train = model.evaluate(x_train, y_train)
score_test = model.evaluate(x_test, y_test)
print(score_train, score_test)
t1 = time.time()
print('total time cost:', t1 - t0)
plot_learning_curves(history)
model.save('/xxx.h5')

import numpy as np
import math
import time
import h5py
import numpy as np
import os
import sys
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import pickle
import sklearn
import pickle
from numpy import array
from keras.layers import Permute
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import Input
from tensorflow.keras.models import Model
from sklearn.model_selection import train_test_split
from tensorflow.keras import optimizers
from tensorflow import losses
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, 'Not enough GPU hardware devices available'
tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(tf.__version__)
print(sys.version_info)
for module in (mpl, np, pd, sklearn, tf, keras):
    print(module.__name__, module.__version__)
folder = '/waveform_example'
folder
dataset_y = []
dataset_x = []
for root, dirs, files in os.walk(folder):
    for filename in files:
        if filename.endswith('.hdf5'):
            print(filename)
            f = h5py.File(folder + '/' + filename, 'r')
            liv_1 = list(f.keys())
            for I in liv_1:
                data = f[I]
                liv_2 = list(data.keys())
                for ch in liv_2:
                    if ch == 'CH_A':
                        print(ch)
                        data[ch].keys()
                        liv_3 = list(data[ch].keys())
                        for trg in liv_3:
                            if max(data[ch][trg][()]) < 75000:
                                x = data[ch][trg][()]
                            if 'Am' in filename:
                                dataset_y.append(1)
                            else:
                                dataset_y.append(0)
                            dataset_x.append(x)
X = dataset_x.copy()
print(X)
X = [(-i) for i in X]
print(X)
y = dataset_y.copy()
np.random.seed(12)
np.random.shuffle(X)
np.random.seed(12)
np.random.shuffle(y)
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
x_train, x_valid, y_train, y_valid = train_test_split(x_train, y_train,
    test_size=0.2)
x_train = np.array(x_train)
y_train = np.array(y_train)
x_valid = np.array(x_valid)
y_valid = np.array(y_valid)
x_test = np.array(x_test)
y_test = np.array(y_test)
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
x_valid = np.reshape(x_valid, (x_valid.shape[0], x_valid.shape[1], 1))
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
print(x_train.shape, y_train.shape, x_valid.shape, y_valid.shape, x_test.
    shape, y_test.shape)
t0 = time.time()
input_fca = Input(shape=(16384, 1))
l1 = tf.keras.layers.GRU(units=32, dropout=0.5, return_sequences=True)(
    input_fca)
print(l1)
l2 = keras.layers.Bidirectional(keras.layers.GRU(units=64, dropout=0.7,
    return_sequences=False))(l1)
l3 = tf.keras.layers.Dense(64, activation='selu')(l2)
l4 = tf.keras.layers.Dense(32, activation='selu')(l3)
output__layer = tf.keras.layers.Dense(1, activation='sigmoid')(l4)
model = Model(inputs=[input_fca], outputs=[output__layer])
model.summary()
initia_learning_rate = 0.01
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initia_learning_rate, decay_steps=899, decay_rate=0.1, staircase=False)
model.compile(optimizer=keras.optimizers.Adam(learning_rate=lr_schedule),
    loss=tf.losses.BinaryCrossentropy(), metrics=['accuracy'])
history = model.fit(x_train, y_train, validation_data=[x_valid, y_valid],
    epochs=10, batch_size=64)


def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize=(8, 5))
    plt.grid(True)
    plt.gca().set_ylim(0, 2)
    plt.savefig('/xxx.png')
    plt.show()


score_train = model.evaluate(x_train, y_train)
score_test = model.evaluate(x_test, y_test)
print(score_train, score_test)
t1 = time.time()
print('total time cost:', t1 - t0)
plot_learning_curves(history)
model.save('/xxx.h5')

### Data extraction
import numpy as np
import math
import time
import h5py
import numpy as np
import os
import sys
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import pickle
import sklearn
import pickle
from numpy import array
from keras.layers import Permute
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import Input
from tensorflow.keras.models import Model
from sklearn.model_selection import train_test_split
from tensorflow.keras import optimizers
from tensorflow import losses

physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
tf.config.experimental.set_memory_growth(physical_devices[0], True)




print(tf.__version__)
print(sys.version_info)
for module in mpl,np,pd,sklearn,tf,keras :
     print(module.__name__,module.__version__)
folder = '/waveform_example'
folder

###Data preparation script
dataset_y = []
dataset_x = []
for root,dirs,files in os.walk(folder):
    #scorri i file dentro la cartella
    for filename in files:
    #controlla che l'estensione del file sia .hdf5
     if filename.endswith((".hdf5")):
        print(filename)
        #create_dir(folder+'/'+filename)
        #naviga i deversi livelli del file
        f = h5py.File(folder+'/'+filename,'r')
        liv_1 = list(f.keys())
        for I in liv_1:
            data = f[I]
            liv_2 = list(data.keys())
            for ch in liv_2:
            #Considera solo i dati che al livello 2 hanno CH_A
                    if ch == 'CH_A':
                       print(ch)
                       data[ch].keys()
                       liv_3 = list(data[ch].keys())
                       for trg in liv_3:
                    #print(trg)
                    #Estrai la serie storica
                          if max(data[ch][trg][()])<75000:
                             x = data[ch][trg][()]
                           #Agguingi come ultimo elemento il valore della variabile
                           #target per la serie storica sulla base del nome del file di partenza
                          if 'Am' in filename:
                             dataset_y.append(1)
                          else:
                             dataset_y.append(0)
                    #Appendi alla lista "dataset" il valore della serei storica con relativa etichetta
                          dataset_x.append(x)
    #Salvare ne lformato che si ritiene piu comodo
X = dataset_x.copy()
print(X)

X = [-i for i in X]
print(X)
y = dataset_y.copy()

np.random.seed(12)
np.random.shuffle(X)
np.random.seed(12)
np.random.shuffle(y)


x_train,x_test,y_train,y_test = train_test_split(X,y,test_size = 0.1)
x_train,x_valid,y_train,y_valid = train_test_split(x_train,y_train,test_size = 0.2)

x_train = np.array(x_train)
y_train = np.array(y_train)
x_valid = np.array(x_valid)
y_valid = np.array(y_valid)
x_test = np.array(x_test)
y_test = np.array(y_test)



x_train= np.reshape(x_train,(x_train.shape[0],x_train.shape[1],1))
x_valid= np.reshape(x_valid,(x_valid.shape[0],x_valid.shape[1],1))
x_test = np.reshape(x_test,(x_test.shape[0],x_test.shape[1],1))



print(x_train.shape,
y_train.shape,
x_valid.shape,
y_valid.shape,
x_test.shape,
y_test.shape)

t0 = time.time()

input_fca = Input(shape=(16384,1))





l1 = tf.keras.layers.GRU(units = 32,dropout= 0.5,return_sequences=True)(input_fca)
print(l1)

l2 = keras.layers.Bidirectional(keras.layers.GRU(units=64,dropout = 0.7,return_sequences=False))(l1)

l3 = tf.keras.layers.Dense(64,activation= 'selu')(l2)

l4 = tf.keras.layers.Dense(32,activation= 'selu')(l3)

output__layer = tf.keras.layers.Dense(1,activation= 'sigmoid')(l4)



model = Model(inputs=[input_fca], outputs=[output__layer])

model.summary()

initia_learning_rate = 0.01

lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
    initia_learning_rate,
    decay_steps=899,
    decay_rate=0.1,
    staircase=False)
model.compile(optimizer= keras.optimizers.Adam(learning_rate=lr_schedule),loss = tf.losses.BinaryCrossentropy(), metrics= ['accuracy'])

history = model.fit(x_train, y_train,validation_data= ([x_valid,y_valid]),epochs=10,batch_size= 64)
def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize = (8,5))
    plt.grid(True)
    plt.gca().set_ylim(0,2)
    plt.savefig("/xxx.png")
    plt.show()

score_train = model.evaluate(x_train,y_train)
score_test = model.evaluate(x_test,y_test)
print(score_train,score_test)


t1 = time.time()
print('total time cost:',t1-t0)

plot_learning_curves(history)




model.save('/xxx.h5')