PLSR

Durante il dottorato avevo la PLSR (Partial Least Square Regression) per analizzare i dati di dottorato ma senza gran successo...(in generale il modello fittava bene in fase di training ma esplodeva in fase di validazione)

Usando i dati del precedente post sull'umidita' ho provato a modellizzare con PLSR ovviamente con l'aiuto di Gemini AI

Il numero otttimale di componenti e' risultato di 9 

 

R2=0.88

 

 

 

import pandas as pd

from sklearn.cross_decomposition import PLSRegression

from sklearn.model_selection import train_test_split

from sklearn.metrics import mean_squared_error, r2_score

df = pd.read_csv('/content/soilmoisture_dataset.csv')

# Extract Y values (3rd column, index 2)

Y = df.iloc[:, 2] # Assuming the 3rd column (index 2) is 'soil_moisture'

# Extract X values (columns from 5th to the end, index 4 onwards)

X = df.iloc[:, 4:] # Assuming columns from index 4 onwards are reflectance values

# Split data into training and testing sets (optional, but good practice for model evaluation)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)

# Initialize and train the PLSR model

# The number of components (n_components) is a hyperparameter that might need tuning.

# A common starting point is to use a small number, e.g., 2 or 3, or determine it through cross-validation.

pls = PLSRegression(n_components=9) # Changed to 4 components as requested

pls.fit(X_train, Y_train)

print("\nPLSR model trained successfully!")

# Make predictions on the test set

Y_pred = pls.predict(X_test)

# Evaluate the model

mse = mean_squared_error(Y_test, Y_pred)

r2 = r2_score(Y_test, Y_pred)

print(f"Mean Squared Error (MSE): {mse:.4f}")

print(f"R-squared (R2) score: {r2:.4f}")

from sklearn.model_selection import KFold, cross_val_score

import numpy as np

import matplotlib.pyplot as plt

max_components = min(X_train.shape[1], X_train.shape[0] - 1)

components_to_test = np.arange(1, min(max_components + 1, 16))

mse_scores = []

r2_scores = []

kf = KFold(n_splits=5, shuffle=True, random_state=42)

print(f"Tuning n_components from 1 to {components_to_test[-1]} using 5-Fold Cross-Validation...")

for n_comp in components_to_test:

pls_cv = PLSRegression(n_components=n_comp)

# Calculate negative mean squared error (sklearn's cross_val_score minimizes, so use negative MSE)

# And R-squared score

nmse = cross_val_score(pls_cv, X_train, Y_train, cv=kf, scoring='neg_mean_squared_error')

r2 = cross_val_score(pls_cv, X_train, Y_train, cv=kf, scoring='r2')

mse_scores.append(-np.mean(nmse)) # Convert back to positive MSE

r2_scores.append(np.mean(r2))

# Find the optimal n_components based on the minimum MSE

best_n_components_mse = components_to_test[np.argmin(mse_scores)]

min_mse = np.min(mse_scores)

# Find the optimal n_components based on the maximum R2

best_n_components_r2 = components_to_test[np.argmax(r2_scores)]

max_r2 = np.max(r2_scores)

print(f"\nOptimal n_components (min MSE): {best_n_components_mse} (MSE: {min_mse:.4f})")

print(f"Optimal n_components (max R2): {best_n_components_r2} (R2: {max_r2:.4f})")

# Plotting the results

plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)

plt.plot(components_to_test, mse_scores, marker='o')

plt.title('PLSR MSE vs. Number of Components (Cross-Validation)')

plt.xlabel('Number of Components')

plt.ylabel('Mean Squared Error (MSE)')

plt.xticks(components_to_test)

plt.grid(True)

plt.axvline(x=best_n_components_mse, color='r', linestyle='--', label=f'Optimal: {best_n_components_mse}')

plt.legend()

plt.subplot(1, 2, 2)

plt.plot(components_to_test, r2_scores, marker='o', color='green')

plt.title('PLSR R-squared vs. Number of Components (Cross-Validation)')

plt.xlabel('Number of Components')

plt.ylabel('R-squared (R2)')

plt.xticks(components_to_test)

plt.grid(True)

plt.axvline(x=best_n_components_r2, color='r', linestyle='--', label=f'Optimal: {best_n_components_r2}')

plt.legend()

plt.tight_layout()

plt.show()

# Retrain the model with the best n_components and evaluate on the test set

final_pls_model = PLSRegression(n_components=best_n_components_r2) # Using R2 optimized components

final_pls_model.fit(X_train, Y_train)

Y_pred_tuned = final_pls_model.predict(X_test)

mse_tuned = mean_squared_error(Y_test, Y_pred_tuned)

r2_tuned = r2_score(Y_test, Y_pred_tuned)

print(f"\nModel re-trained with optimal n_components ({best_n_components_r2}).")

print(f"Test Set Mean Squared Error (MSE) after tuning: {mse_tuned:.4f}")

print(f"Test Set R-squared (R2) score after tuning: {r2_tuned:.4f}")

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(8, 6))

sns.scatterplot(x=Y_test, y=Y_pred_tuned.flatten(), color='blue', alpha=0.7)

plt.plot([min(Y_test), max(Y_test)], [min(Y_test), max(Y_test)], color='red', linestyle='--', lw=2, label='Perfect Prediction')

plt.title('Actual vs. Predicted Soil Moisture (Tuned PLSR Model)')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.grid(True)

plt.legend()

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

# Get the names of the top 10 predictor variables

top_10_features = sorted_importance.head(10).index

# Set up the figure and axes for subplots

fig, axes = plt.subplots(nrows=3, ncols=4, figsize=(20, 15))

axes = axes.flatten() # Flatten the 2D array of axes for easy iteration

fig.suptitle('Relationship Between Top Predictor Wavelengths and Soil Moisture', fontsize=16)

# Iterate through the top 10 features and create a scatter plot for each

for i, feature in enumerate(top_10_features):

sns.scatterplot(x=X_train[feature], y=Y_train, ax=axes[i], alpha=0.6)

axes[i].set_title(f'Wavelength: {feature} nm')

axes[i].set_xlabel(f'Reflectance at {feature} nm')

axes[i].set_ylabel('Soil Moisture')

axes[i].grid(True, linestyle='--', alpha=0.7)

# Hide any unused subplots

for j in range(len(top_10_features), len(axes)):

fig.delaxes(axes[j])

plt.tight_layout(rect=[0, 0.03, 1, 0.95]) # Adjust layout to prevent title overlap

plt.show()

import matplotlib.pyplot as plt

import numpy as np

# Get original wavelengths from X_original column names

original_wavelengths = X_original.columns.astype(float).tolist()

# Calculate mid-points for derivative wavelengths

derivative_wavelengths = []

for i in range(len(original_wavelengths) - 1):

mid_point = (original_wavelengths[i] + original_wavelengths[i+1]) / 2

derivative_wavelengths.append(mid_point)

# Convert derivative_wavelengths to a numpy array for plotting

derivative_wavelengths = np.array(derivative_wavelengths)

plt.figure(figsize=(12, 7))

# Plot all derivative spectra

for i in range(X_deriv.shape[0]):

plt.plot(derivative_wavelengths, X_deriv.iloc[i], alpha=0.5, color='blue') # Use a single color and slight transparency

plt.title('All First Derivative of Reflectance Spectra')

plt.xlabel('Wavelength (nm)')

plt.ylabel('First Derivative of Reflectance')

plt.grid(True, linestyle='--', alpha=0.7)

# Removed legend as plotting all samples makes it unreadable

plt.tight_layout()

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(8, 6))

sns.scatterplot(x=Y_test, y=Y_pred.flatten(), color='green', alpha=0.7)

plt.plot([min(Y_test), max(Y_test)], [min(Y_test), max(Y_test)], color='red', linestyle='--', lw=2, label='Perfect Prediction')

plt.title('Actual vs. Predicted Soil Moisture (Derivative Model)')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.grid(True)

plt.legend()

plt.show()

# Extract Y values (3rd column, index 2)

Y = df.iloc[:, 2] # Assuming the 3rd column (index 2) is 'soil_moisture'

# Extract X values (columns from 5th to the end, index 4 onwards) - original reflectance

X_original = df.iloc[:, 4:] # Assuming columns from index 4 onwards are reflectance values

# Calculate the first derivative of reflectance

# Using np.diff along axis=1 (columns) for each row (sample)

X_deriv = pd.DataFrame(np.diff(X_original.values, axis=1))

# The new column names will correspond to the mid-points or just be sequential

# For simplicity, let's create generic names for the derivative columns

derivative_col_names = [f'deriv_{col_idx}' for col_idx in range(X_deriv.shape[1])]

X_deriv.columns = derivative_col_names

# Now, use the derivative as the feature matrix X

X = X_deriv

# Display the shapes of X and Y to confirm

print(f"Shape of X (first derivative reflectance values): {X.shape}")

print(f"Shape of Y (soil_moisture): {Y.shape}")

# Split data into training and testing sets (optional, but good practice for model evaluation)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)

# Initialize and train the PLSR model

# The number of components (n_components) is a hyperparameter that might need tuning.

# A common starting point is to use a small number, e.g., 2 or 3, or determine it through cross-validation.

pls = PLSRegression(n_components=4) # Changed to 4 components as requested

pls.fit(X_train, Y_train)

print("\nPLSR model trained successfully!")

# Make predictions on the test set

Y_pred = pls.predict(X_test)

# Evaluate the model

mse = mean_squared_error(Y_test, Y_pred)

r2 = r2_score(Y_test, Y_pred)

print(f"Mean Squared Error (MSE): {mse:.4f}")

print(f"R-squared (R2) score: {r2:.4f}")

"""The predictor variables are represented by column names that are numerical, likely corresponding to specific wavelengths (in nanometers) from the reflectance spectrum. These wavelengths are the features identified by the PLSR model as having the strongest influence on predicting soil moisture.

**Interpreting the Coefficients:**

The regression coefficients obtained from the PLSR model quantify the relationship between each predictor variable (wavelength) and the target variable (soil moisture).:

* **Magnitude:** The absolute value of a coefficient indicates the strength of the variable's influence. A larger absolute coefficient means that a small change in that wavelength's reflectance has a greater impact on the predicted soil moisture.

* **Sign (Positive/Negative):**

* A **positive coefficient** indicates a positive correlation: as the reflectance at that wavelength increases, the soil moisture is predicted to increase.

* A **negative coefficient** indicates a negative correlation: as the reflectance at that wavelength increases, the soil moisture is predicted to decrease.

In the context of soil moisture prediction from reflectance data, specific wavelengths are known to be more sensitive to water absorption. The model has highlighted these critical wavelengths, allowing us to understand which parts of the spectrum are most informative for estimating soil moisture content.

Based on the analysis, the top 10 most important predictor variables (wavelengths) and their respective absolute coefficient values are:

"""

display(sorted_importance.head(10))

 

 

 

 

 

Errore NTFS su Debian

Montando un disco USB esterno che usiamo di scambio a lavoro (e quindi formattato NTFS per Windows) e' comparso il seguente errore 

 

luca@HW27747:~$ sudo mount -t ntfs-3g /dev/sda1 /media/cdrom
$MFTMirr does not match $MFT (record 3).
Failed to mount '/dev/sda1': Input/output error
NTFS is either inconsistent, or there is a hardware fault, or it's a
SoftRAID/FakeRAID hardware. In the first case run chkdsk /f on Windows
then reboot into Windows twice. The usage of the /f parameter is very
important! If the device is a SoftRAID/FakeRAID then first activate
it and mount a different device under the /dev/mapper/ directory, (e.g.
/dev/mapper/nvidia_eahaabcc1). Please see the 'dmraid' documentation
for more details. 

la soluzione e' banale 

sudo ntfsfix /dev/sda1 

Soil Moisture con dati iperspettrali

Durante il dottorato avevo fatto qualche prova con campioni artificiale di terreno ad umidita' controllata ed avevo una ottima correlazione con la misura iperspettrale

I campioni erano di terreno naturale con umidita' aggiunta artificialmente 

Campione di tesi dottorato - illuminazione solare

 

Dopo la rimozione del continuum avevo lavorato sugli assorbimenti nello swir
 

Adesso ho trovato questo lavoro

Felix M. Riese and Sina Keller, “Introducing a Framework of Self-Organizing Maps for Regression of Soil Moisture with Hyperspectral Data,” in IGARSS 2018 - 2018 IEEE International Geoscience and Remote Sensing Symposium, Valencia, Spain, 2018, pp. 6151-6154. 

i dati sono riportati a questo link in formato csv con 680 spettri VNIR di campagna e relative misure di umidita' realizzate in sito tramite TRIME-PICO time-domain reflectometry (TDR) 

questi sono i dati normalizzati con rimozione del continuo

 In VNIR non risulta immediato un approccio di misura della profondita' di picco ed ho provato a vedere come si comportava un approccio random forest (ho utilizzato gemini in Google Colab in modo da avere anche l'ottimizzazione dei parametri in modo semplice)

import pandas as pd

# Load the dataset

df = pd.read_csv('/content/soilmoisture_dataset.csv')

display(df.head())

# The third column (index 2) is Y

Y = df.iloc[:, 2]

# Columns from the 19th (index 18) to the end are X

X = df.iloc[:, 18:]

# Drop rows with any NaN values in X or Y for model training

data = pd.concat([X, Y], axis=1).dropna()

X = data.iloc[:, :-1]

Y = data.iloc[:, -1]

print(f"Shape of X: {X.shape}")

print(f"Shape of Y: {Y.shape}")

display(X.head())

display(Y.head())

from sklearn.model_selection import train_test_split

from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_squared_error, r2_score

# Split data into training and testing sets

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)

# Initialize and train the Random Forest Regressor model

model = RandomForestRegressor(n_estimators=100, random_state=42)

model.fit(X_train, Y_train)

# Make predictions on the test set

Y_pred = model.predict(X_test)

# Evaluate the model

mse = mean_squared_error(Y_test, Y_pred)

r2 = r2_score(Y_test, Y_pred)

print(f"Mean Squared Error: {mse:.4f}")

print(f"R-squared: {r2:.4f}")

from sklearn.model_selection import RandomizedSearchCV

from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import mean_squared_error, r2_score

import numpy as np

# Define the parameter distribution for RandomizedSearchCV

param_dist = {

'n_estimators': np.arange(100, 1000, 100), # Number of trees in the forest

'max_features': ['sqrt', 'log2'], # Number of features to consider at every split

'max_depth': [None, 10, 20, 30, 40, 50], # Maximum number of levels in tree

'min_samples_split': np.arange(2, 20, 2), # Minimum number of samples required to split a node

'min_samples_leaf': np.arange(1, 10, 1), # Minimum number of samples required at each leaf node

'bootstrap': [True, False] # Method for sampling data points (with or without replacement)

}

# Initialize a base Random Forest Regressor

rf = RandomForestRegressor(random_state=42)

# Initialize RandomizedSearchCV

# n_iter controls the number of parameter settings that are sampled.

# cv is the number of folds for cross-validation.

random_search = RandomizedSearchCV(estimator=rf, param_distributions=param_dist,

n_iter=50, cv=5, verbose=2, random_state=42,

n_jobs=-1, scoring='neg_mean_squared_error')

# Fit the random search model

random_search.fit(X_train, Y_train)

print("Best Parameters found:", random_search.best_params_)

print("Best Negative Mean Squared Error (from CV):", random_search.best_score_)

# Get the best model

best_rf_model = random_search.best_estimator_

# Make predictions with the best model on the test set

Y_pred_tuned = best_rf_model.predict(X_test)

# Evaluate the tuned model

mse_tuned = mean_squared_error(Y_test, Y_pred_tuned)

r2_tuned = r2_score(Y_test, Y_pred_tuned)

print(f"\nPerformance of the tuned Random Forest model on the test set:")

print(f"Mean Squared Error (tuned): {mse_tuned:.4f}")

print(f"R-squared (tuned): {r2_tuned:.4f}")

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(10, 7))

sns.scatterplot(x=Y_test, y=Y_pred, alpha=0.6)

plt.plot([Y.min(), Y.max()], [Y.min(), Y.max()], 'r--', lw=2, label='Perfect Prediction Line') # Add a diagonal line for perfect prediction

plt.title('Actual vs. Predicted Soil Moisture Values (Test Set)')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.legend()

plt.grid(True)

plt.tight_layout()

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(10, 7))

sns.scatterplot(x=Y_test, y=Y_pred_tuned, alpha=0.6)

plt.plot([Y.min(), Y.max()], [Y.min(), Y.max()], 'r--', lw=2, label='Perfect Prediction Line')

plt.title('Actual vs. Predicted Soil Moisture Values (Tuned Model - Test Set)')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.legend()

plt.grid(True)

plt.tight_layout()

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(10, 7))

sns.scatterplot(x=Y_test, y=Y_pred_tuned, alpha=0.6)

plt.plot([Y.min(), Y.max()], [Y.min(), Y.max()], 'r--', lw=2, label='Perfect Prediction Line')

plt.title('Actual vs. Predicted Soil Moisture Values (Tuned Model - Test Set)')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.legend()

plt.grid(True)

plt.tight_layout()

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(10, 7))

sns.scatterplot(x=Y_test, y=Y_pred, alpha=0.6)

plt.plot([Y.min(), Y.max()], [Y.min(), Y.max()], 'r--', lw=2, label='Perfect Prediction Line') # Add a diagonal line for perfect prediction

plt.title('Actual vs. Predicted Soil Moisture Values (Original Model - Test Set)')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.legend()

plt.grid(True)

plt.tight_layout()

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

plt.figure(figsize=(10, 6))

sns.histplot(residuals_tuned, kde=True)

plt.title('Distribution of Residuals for Tuned Model')

plt.xlabel('Residuals (Actual - Predicted)')

plt.ylabel('Frequency')

plt.grid(True)

plt.tight_layout()

plt.show()

residuals_tuned = Y_test - Y_pred_tuned

print("First 5 residuals of the tuned model:")

display(residuals_tuned.head())

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

# Get feature importances from the model

feature_importances = model.feature_importances_

# Create a DataFrame for better visualization

feature_names = X.columns

importance_df = pd.DataFrame({'Feature': feature_names, 'Importance': feature_importances})

# Sort the features by importance in descending order

importance_df = importance_df.sort_values(by='Importance', ascending=False)

# Display the top 10 most important features

print("Top 10 Most Important Features:")

display(importance_df.head(10))

# Plot the top 20 most important features

plt.figure(figsize=(12, 8))

sns.barplot(x='Importance', y='Feature', data=importance_df.head(20), palette='viridis', hue='Feature', legend=False)

plt.title('Top 20 Most Important Features (Random Forest Regressor)')

plt.xlabel('Importance')

plt.ylabel('Feature')

plt.tight_layout()

plt.show()

Questo e' il grafico del modello non ottimizzato

e questo il grafico ottimizzato

grafico degli errori residui

 

 Come si vede il modello funziona deciamente bene considerando che siamo in condizioni naturali.L'aspetto decisamente piu' interessante e' dal grafico successivo sul peso delle diverse bande nei dati

Come si vede viene estratto il valore di 950 nm come lunghezza d'onda parametro piu' significativo del modello (che e' anche l'estremo del sensore)...questa lunghezza d'onda corrisponde (in realta' sarebbe a 970 nm ma al di fuori del range di misura) della terza armonica del legame O-H. Quindi il modello in maniera autonoma ha individuata una variabile latente (la terza armonica e' un assorbimento molto piu' debole rispetto a quelli presenti nello swir)

Visto che in in linea di principio la posizione dell'assorbimento e' anche funzione della temperatura e visto che il dataset include anche la temperatura del suolo ho provato ad inserire nei valori X del modello anche la temperatura oltre alla riflettanza

 

Non me lo aspettavo ma il modello e' migliorato sensibilmente. 
 

Questo il codice modificato

import pandas as pd

df = pd.read_csv('/content/soilmoisture_dataset.csv')

print("First 5 rows of the dataset:")

display(df.head())

print("\nInformation about the dataset:")

df.info()

y_columns = ['soil_moisture']

X_columns = ['soil_temperature'] + [col for col in df.columns if col.isdigit()]

X = df[X_columns]

Y = df[y_columns]

print(f"Shape of features (X): {X.shape}")

print(f"Shape of targets (Y): {Y.shape}")

print("\nFirst 5 rows of features (X):")

display(X.head())

print("\nFirst 5 rows of targets (Y):")

display(Y.head())

from sklearn.model_selection import train_test_split

from sklearn.ensemble import RandomForestRegressor

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)

print(f"X_train shape: {X_train.shape}")

print(f"X_test shape: {X_test.shape}")

print(f"Y_train shape: {Y_train.shape}")

print(f"Y_test shape: {Y_test.shape}")

model = RandomForestRegressor(random_state=42)

model.fit(X_train, Y_train.values.ravel()) # .values.ravel() to flatten Y_train for single output regression

Y_pred = model.predict(X_test)

print("\nFirst 5 predictions:")

print(Y_pred[:5])

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

# Get feature importances

feature_importances = model.feature_importances_

# Create a DataFrame for better visualization

importance_df = pd.DataFrame({

'Feature': X_columns,

'Importance': feature_importances

})

# Sort by importance in descending order

importance_df = importance_df.sort_values(by='Importance', ascending=False)

print("Top 10 Feature Importances:")

display(importance_df.head(10))

# Visualize top N feature importances

plt.figure(figsize=(12, 7))

sns.barplot(x='Importance', y='Feature', data=importance_df.head(10))

plt.title('Top 10 Feature Importances for Soil Moisture and Temperature Prediction')

plt.xlabel('Importance')

plt.ylabel('Reflectance Band (Feature)')

plt.show()

print("Bottom 10 Feature Importances:")

display(importance_df.tail(10))

# Visualize bottom N feature importances

plt.figure(figsize=(12, 7))

sns.barplot(x='Importance', y='Feature', data=importance_df.tail(10))

plt.title('Bottom 10 Feature Importances for Soil Moisture and Temperature Prediction')

plt.xlabel('Importance')

plt.ylabel('Reflectance Band (Feature)')

plt.show()

import matplotlib.pyplot as plt

import seaborn as sns

# Create a DataFrame for actual vs. predicted values for easier plotting

results_df = Y_test.copy()

results_df['soil_moisture_pred'] = Y_pred # Y_pred is now a 1D array for single output

# Calculate residuals

results_df['soil_moisture_residuals'] = results_df['soil_moisture'] - results_df['soil_moisture_pred']

# Plotting Actual vs. Predicted for Soil Moisture

plt.figure(figsize=(12, 6))

sns.scatterplot(x='soil_moisture', y='soil_moisture_pred', data=results_df)

plt.plot([results_df['soil_moisture'].min(), results_df['soil_moisture'].max()],

[results_df['soil_moisture'].min(), results_df['soil_moisture'].max()],

color='red', linestyle='--', lw=2)

plt.title('Actual vs. Predicted Soil Moisture')

plt.xlabel('Actual Soil Moisture')

plt.ylabel('Predicted Soil Moisture')

plt.grid(True, linestyle='--', alpha=0.7)

plt.show()

# Plotting Residuals for Soil Moisture

plt.figure(figsize=(12, 6))

sns.scatterplot(x=results_df['soil_moisture_pred'], y=results_df['soil_moisture_residuals'])

plt.axhline(y=0, color='r', linestyle='--')

plt.title('Residual Plot for Soil Moisture')

plt.xlabel('Predicted Soil Moisture')

plt.ylabel('Residuals (Actual - Predicted)')

plt.grid(True, linestyle='--', alpha=0.7)

plt.show()

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

# Since Y_pred is now a 1D array for a single target, we don't need to iterate or use Y_pred[:, i]

print(f"\n--- Metrics for {y_columns[0]} ---")

mae = mean_absolute_error(Y_test, Y_pred)

mse = mean_squared_error(Y_test, Y_pred)

r2 = r2_score(Y_test, Y_pred)

print(f"Mean Absolute Error (MAE): {mae:.3f}")

print(f"Mean Squared Error (MSE): {mse:.3f}")

print(f"R-squared (R2) Score: {r2:.3f}")

 

 visto che fare esperimenti usando la AI e' estremamente veloce ho provato ad elaborare i dati usando una 1D-CNN. I risultati sono stati pessimi 

 

import pandas as pd

import numpy as np

from sklearn.model_selection import train_test_split

from tensorflow.keras.models import Sequential

from tensorflow.keras.layers import Conv1D, MaxPooling1D, Flatten, Dense

"""### Load the dataset"""

df = pd.read_csv('/content/soilmoisture_dataset.csv', header=0)

display(df.head())

"""### Extract x and y values"""

# The third column is y (index 2)

y = df.iloc[:, 2].values

# Columns from 5 to the end are x values (reflectance)

x = df.iloc[:, 4:].values

print(f"Shape of x: {x.shape}")

print(f"Shape of y: {y.shape}")

"""### Preprocess data for 1D CNN"""

# Reshape x for 1D CNN (samples, timesteps, features)

# In our case, timesteps are the reflectance values and features is 1

x = x.reshape(x.shape[0], x.shape[1], 1)

print(f"Shape of x after reshaping: {x.shape}")

# Split data into training and testing sets

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)

print(f"Shape of x_train: {x_train.shape}")

print(f"Shape of y_train: {y_train.shape}")

print(f"Shape of x_test: {x_test.shape}")

print(f"Shape of y_test: {y_test.shape}")

"""### Create and train a 1D CNN model"""

model = Sequential([

Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(x.shape[1], 1)),

MaxPooling1D(pool_size=2),

Conv1D(filters=128, kernel_size=3, activation='relu'),

MaxPooling1D(pool_size=2),

Flatten(),

Dense(50, activation='relu'),

Dense(1) # Output layer for regression

])

model.compile(optimizer='adam', loss='mse')

model.summary()

history = model.fit(x_train, y_train, epochs=50, batch_size=32, validation_split=0.2)

# Evaluate the model

loss = model.evaluate(x_test, y_test)

print(f"Test Loss: {loss}")

import matplotlib.pyplot as plt

# Plot training & validation loss values

plt.figure(figsize=(10, 6))

plt.plot(history.history['loss'], label='Train Loss')

plt.plot(history.history['val_loss'], label='Validation Loss')

plt.title('Model Loss over Epochs')

plt.ylabel('Loss (Mean Squared Error)')

plt.xlabel('Epoch')

plt.legend(loc='upper right')

plt.grid(True)

plt.show()

y_pred = model.predict(x_test)

print(f"Shape of predictions: {y_pred.shape}")

print("First 5 predictions:")

for i in range(5):

print(f"Predicted: {y_pred[i][0]:.2f}, Actual: {y_test[i]:.2f}")

from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

# Make predictions with the re-trained model

y_pred = model.predict(x_test)

# Calculate Mean Absolute Error (MAE)

mae = mean_absolute_error(y_test, y_pred)

print(f"Mean Absolute Error (MAE): {mae:.2f}")

# Calculate Mean Squared Error (MSE)

mse = mean_squared_error(y_test, y_pred)

print(f"Mean Squared Error (MSE): {mse:.2f}")

# Calculate R-squared (R2 Score)

r2 = r2_score(y_test, y_pred)

print(f"R-squared (R2 Score): {r2:.2f}")

import matplotlib.pyplot as plt

plt.figure(figsize=(10, 6))

plt.scatter(y_test, y_pred)

plt.xlabel('Actual Values')

plt.ylabel('Predicted Values')

plt.title('Actual vs. Predicted Soil Moisture Values (MLP Model)')

plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=2, color='red') # Diagonal line

plt.grid(True)

plt.show()