Использование фреймворка Hydra с Compose-API

import os
import gzip

import pandas as pd
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

from hydra import compose, initialize
from hydra.core.global_hydra import GlobalHydra
from omegaconf import OmegaConf

GlobalHydra.instance().clear()

# Hydra context initialization
initialize(version_base=None, config_path=".", job_name="trees-training-getparams")
cfg = compose(config_name="config")

print(OmegaConf.to_yaml(cfg))

default:
- _self_
paths:
  data: ../data
  artifacts: ../docs/artifacts
files:
  train_data: tree-data-clean-half.csv.gz
params:
  validation_split: 0.2
  epochs: 10
  batch_size: 64

# Загружаем датасет
data_file = os.path.join(os.path.join(cfg.paths.data, cfg.files.train_data))
with gzip.open(data_file, 'rb') as gz_file:
    data = pd.read_csv(gz_file)

print(f'Dataset has been loaded. Shape of data: {data.shape}')

Dataset has been loaded. Shape of data: (321504, 42)

# Предобработка данных
data = data[['tree_dbh', 'spc_common', 'health']]
data = data.dropna()

# Кодирование категориальных признаков
label_encoder = LabelEncoder()
data['spc_common'] = label_encoder.fit_transform(data['spc_common'])
data['health'] = label_encoder.fit_transform(data['health'])

# Разделение данных на train и test
X = data[['tree_dbh', 'spc_common']]
y = data['health']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Нормализация числовых признаков
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Создание модели
model = keras.Sequential([
    layers.Dense(64, activation='relu', input_shape=(2,)),
    layers.Dense(32, activation='relu'),
    layers.Dense(3, activation='softmax')
])

# Компиляция модели
model.compile(optimizer='adam',
            loss='sparse_categorical_crossentropy',
            metrics=['accuracy'])

# Обучение модели
history = model.fit(X_train_scaled, y_train, 
                    epochs=cfg.params.epochs, 
                    batch_size=cfg.params.batch_size, 
                    validation_split=cfg.params.validation_split
)

# Оценка модели
test_loss, test_acc = model.evaluate(X_test_scaled, y_test)
print(f'Training is complete. Accuracy: {test_acc:.3f}')

Epoch 1/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6088 - accuracy: 0.8019 - val_loss: 0.6065 - val_accuracy: 0.8010
Epoch 2/10
3216/3216 [==============================] - 6s 2ms/step - loss: 0.6034 - accuracy: 0.8024 - val_loss: 0.6057 - val_accuracy: 0.8010
Epoch 3/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6028 - accuracy: 0.8024 - val_loss: 0.6047 - val_accuracy: 0.8010
Epoch 4/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6026 - accuracy: 0.8024 - val_loss: 0.6047 - val_accuracy: 0.8010
Epoch 5/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6022 - accuracy: 0.8024 - val_loss: 0.6055 - val_accuracy: 0.8010
Epoch 6/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6020 - accuracy: 0.8024 - val_loss: 0.6047 - val_accuracy: 0.8010
Epoch 7/10
3216/3216 [==============================] - 7s 2ms/step - loss: 0.6019 - accuracy: 0.8024 - val_loss: 0.6035 - val_accuracy: 0.8010
Epoch 8/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6016 - accuracy: 0.8024 - val_loss: 0.6054 - val_accuracy: 0.8010
Epoch 9/10
3216/3216 [==============================] - 8s 2ms/step - loss: 0.6013 - accuracy: 0.8024 - val_loss: 0.6033 - val_accuracy: 0.8010
Epoch 10/10
3216/3216 [==============================] - 7s 2ms/step - loss: 0.6009 - accuracy: 0.8024 - val_loss: 0.6035 - val_accuracy: 0.8010
2010/2010 [==============================] - 4s 2ms/step - loss: 0.6021 - accuracy: 0.8018
Training is complete. Accuracy: 0.802