Introduction to Bash Scripting

#!/bin/bash

# Set variables for directories and file names
input_dir="raw_data"
output_dir="preprocessed_data"
input_file="data.csv"
output_file="cleaned_data.csv"

# Create the output directory if it doesn't exist
mkdir -p $output_dir

# Remove any existing output file to avoid appending to old data
rm -f $output_dir/$output_file

# Add a header to the output file
echo "id,feature1,feature2,feature3,label" > $output_dir/$output_file

# Process the input file line by line
while IFS= read -r line
do
    # Remove any unwanted characters (e.g., '$' and ',')
    cleaned_line=$(echo $line | tr -d '$,')

    # Filter out lines with missing values
    if ! echo $cleaned_line | grep -q "NA"
    then
        # Append the cleaned line to the output file
        echo $cleaned_line >> $output_dir/$output_file
    fi
done < $input_dir/$input_file

echo "Data preprocessing completed. Output file: $output_dir/$output_file"