#!/bin/bash

DIR=/home/phil
HERE=$DIR/TaffB
DATA_DIR=$HERE/data
ERRORS_DIR=$HERE/errors
DATAFILE=$HERE/data.txt
MQTT_CONTROL=$HERE/data_last_loaded_2_mqtt

mkdir -p "$ERRORS_DIR"
touch "$DATAFILE" "$MQTT_CONTROL"

echo "=== Starting Process Data at $(date) ==="

# ---------------------------------------------------------
# Phase 1: Determine cutoff timestamp from existing data.txt
# ---------------------------------------------------------
LAST_DATA_LINE=$(tail -n 1 "$DATAFILE")
CUTOFF_TS=0

if [ -n "$LAST_DATA_LINE" ]; then
    PARSED_TS=$(echo "$LAST_DATA_LINE" | jq -r '.Timestamp // empty' 2>/dev/null)
    if [ -n "$PARSED_TS" ]; then
        CUTOFF_TS=$PARSED_TS
    fi
fi

echo "Extracting new captures with timestamp > $CUTOFF_TS..."

# ---------------------------------------------------------
# Phase 2: OCR and extraction for new capture files
# ---------------------------------------------------------
NEW_DATA_EXTRACTED=0

for img in $(ls "$DATA_DIR"/last.gather.* 2>/dev/null | grep -v '\.txt$' | sort -V); do
    ts=$(basename "$img" | sed 's/last\.gather\.//')

    # Skip files already processed in data.txt
    if [ "$ts" -le "$CUTOFF_TS" ]; then
        continue
    fi

    echo "--- Processing Timestamp: $ts ---"

    # 1. Ensure OCR files exist; run OCR only if missing
    for dpi in 209 700 800; do
        if [ ! -s "${img}-${dpi}.txt" ]; then
            docker run --rm -v "$DATA_DIR":/data robtesseract "/data/$(basename "$img")" "/data/$(basename "$img")-${dpi}" --dpi "$dpi" 2>/dev/null
        fi
    done

    # 2. Extract POWER across 700, 209, and 800 dpi files
    POWER=""
    for f in "${img}-700.txt" "${img}-209.txt" "${img}-800.txt"; do
        if [ -s "$f" ]; then
            POWER=$(grep -iE 'kwh?|kww' "$f" | head -n1 | grep -oE '[-+]?[0-9]+(\.[0-9]+)?')
            [ -n "$POWER" ] && break
        fi
    done

    if [ -z "$POWER" ]; then
        echo "$ts" >> "$ERRORS_DIR/power_no_found"
        echo "Warning: Power not found for $ts, skipping."
        rm -f "${img}-209.txt" "${img}-700.txt" "${img}-800.txt"
        continue
    fi

    # Fix OCR artifact where +nn turns into 4nn (>100) and format with explicit +/- sign (+0, +5, etc.)
    POWER=$(echo "$POWER" | awk '{
        val = $1 + 0;
        if (val > 100) val = val - int(val/100)*100;
        printf "%+d", val
    }')

    # 3. Extract hydro metrics from the 209 dpi file
    TXT_209="${img}-209.txt"
    if [ ! -s "$TXT_209" ]; then
        echo "$ts: missing 209.txt" >> "$ERRORS_DIR/empty_file"
        rm -f "${img}-209.txt" "${img}-700.txt" "${img}-800.txt"
        continue
    fi

    GENERATORSPEED=$(grep -i 'generator' "$TXT_209" | grep -oE '[-+]?[0-9]+(\.[0-9]+)?' | tail -n1)
    REQUESTOPENING=$(grep -i 'request'   "$TXT_209" | grep -oE '[-+]?[0-9]+(\.[0-9]+)?' | tail -n1)
    TURBINE=$(grep -iE 'turbin|openig'   "$TXT_209" | grep -oE '[-+]?[0-9]+(\.[0-9]+)?' | tail -n1)
    WATERLEVEL=$(grep -i 'water level'   "$TXT_209" | grep -oE '[-+]?[0-9]+(\.[0-9]+)?' | tail -n1)
    WATERPRESSURE=$(grep -i 'pressure'   "$TXT_209" | grep -oE '[-+]?[0-9]+(\.[0-9]+)?' | tail -n1)

    # 4. Range corrections / Decimal shift / Explicit sign formatting
    GENERATORSPEED=$(echo "$GENERATORSPEED" | awk '{printf "%d", $1 + 0}')

    REQUESTOPENING=$(echo "$REQUESTOPENING $POWER" | awk '{
        v = $1 + 0;
        p = $2 + 0;
        while (v > 100) { v = v / 10; }
        if (p <= 20 && v > 25) { v = v / 10; }
        printf "%.1f", v
    }')

    TURBINE=$(echo "$TURBINE $POWER" | awk '{
        v = $1 + 0;
        p = $2 + 0;
        while (v > 100) { v = v / 10; }
        if (p <= 20 && v > 25) { v = v / 10; }
        printf "%.1f", v
    }')

    # WaterLevel: 1912m -> +1.912m (3 decimals, explicit sign)
    WATERLEVEL=$(echo "$WATERLEVEL" | awk '{
        v = $1 + 0;
        if (v > 20 || v < -20) v = v / 1000;
        printf "%+.3f", v
    }')

    # WaterPressure: fix missing decimal point (e.g. 236 -> +2.36, explicit sign)
    WATERPRESSURE=$(echo "$WATERPRESSURE" | awk '{
        v = $1 + 0;
        if (v > 25 || v < -25) v = v / 100;
        printf "%+.2f", v
    }')

    # 5. Validate that all required numeric fields are present
    if [ -z "$GENERATORSPEED" ] || [ -z "$REQUESTOPENING" ] || [ -z "$TURBINE" ] || [ -z "$WATERLEVEL" ] || [ -z "$WATERPRESSURE" ]; then
        echo "Missing fields in $ts: G=$GENERATORSPEED, R=$REQUESTOPENING, T=$TURBINE, WL=$WATERLEVEL, WP=$WATERPRESSURE"
        echo "Timestamp:$ts G:$GENERATORSPEED R:$REQUESTOPENING T:$TURBINE WL:$WATERLEVEL WP:$WATERPRESSURE" >> "$ERRORS_DIR/null_found"
        rm -f "${img}-209.txt" "${img}-700.txt" "${img}-800.txt"
        continue
    fi

    # 6. Sanity Range Checks (Log out-of-range warnings)
    echo "$POWER" | awk '{if ($1 < 0 || $1 > 120) exit 1}' || echo "Power out of range ($POWER): $ts" >> "$ERRORS_DIR/out_of_range"
    echo "$REQUESTOPENING" | awk '{if ($1 < 0 || $1 > 100) exit 1}' || echo "RequestOpening out of range ($REQUESTOPENING): $ts" >> "$ERRORS_DIR/out_of_range"
    echo "$TURBINE" | awk '{if ($1 < 0 || $1 > 100) exit 1}' || echo "Turbine out of range ($TURBINE): $ts" >> "$ERRORS_DIR/out_of_range"
    echo "$WATERPRESSURE" | awk '{if ($1 < -1.0 || $1 > 5.0) exit 1}' || echo "WaterPressure out of range ($WATERPRESSURE): $ts" >> "$ERRORS_DIR/out_of_range"
    echo "$WATERLEVEL" | awk '{if ($1 < -5.0 || $1 > 10.0) exit 1}' || echo "WaterLevel out of range ($WATERLEVEL): $ts" >> "$ERRORS_DIR/out_of_range"

    # 7. Construct valid JSON string matching original format
    JSON_RECORD=$(printf '{"Timestamp":%s,"Power":%s,"GeneratorSpeed":%s,"RequestOpening":%s,"Turbine":%s,"WaterLevel":%s,"WaterPressure":%s}' \
        "$ts" "$POWER" "$GENERATORSPEED" "$REQUESTOPENING" "$TURBINE" "$WATERLEVEL" "$WATERPRESSURE")

    echo "$JSON_RECORD"
    echo "$JSON_RECORD" >> "$DATAFILE"
    NEW_DATA_EXTRACTED=$((NEW_DATA_EXTRACTED + 1))

    # 8. Clean up intermediate OCR files
    rm -f "${img}-209.txt" "${img}-700.txt" "${img}-800.txt"
done

echo "Extraction complete. Added $NEW_DATA_EXTRACTED new record(s) to $DATAFILE."

# ---------------------------------------------------------
# Phase 3: Post unposted records from data.txt to MQTT
# ---------------------------------------------------------
echo "Checking for unposted records to send to MQTT..."

NEWRECS=$(mktemp)
FILE2=$(mktemp)
LAST_MQTT_LINE=$(tail -n 1 "$MQTT_CONTROL")

if [ -n "$LAST_MQTT_LINE" ] && grep -Fxq "$LAST_MQTT_LINE" "$DATAFILE"; then
    awk -v target="$LAST_MQTT_LINE" 'found {print} $0 == target {found=1}' "$DATAFILE" > "$NEWRECS"
else
    cp "$DATAFILE" "$NEWRECS"
fi

if [ -s "$NEWRECS" ]; then
    RECS_COUNT=$(wc -l < "$NEWRECS")
    echo "Found $RECS_COUNT unposted record(s) to publish."

    QUALITY_CONFIG="$HERE/quality_checks/config.json"
    if [ -s "$QUALITY_CONFIG" ]; then
        NUMFIELDS=$(jq '.NumFields' "$QUALITY_CONFIG")
        jq '. | length' "$NEWRECS" | sort -u > "$FILE2"

        if [ "$(wc -l < "$FILE2")" -ne 1 ] || [ "$(cat "$FILE2")" -ne "$NUMFIELDS" ]; then
            echo "Quality check failed: Inconsistent or incorrect field counts."
            rm -f "$NEWRECS" "$FILE2"
            exit 2
        fi
    fi

    while IFS= read -r line || [ -n "$line" ]; do
        [ -z "$line" ] && continue
	if $HERE/mqttcli_linux_amd64 pub --host hydromatch.co.uk -i "HM-TBapp" -t "HM/TB" -m "$line"; then
            echo "Posted: $line"
            echo "$line" > "$MQTT_CONTROL"
        else
            echo "MQTT publish failed for $line. Stopping batch to retry next run."
            rm -f "$NEWRECS" "$FILE2"
            exit 1
        fi
    done < "$NEWRECS"
else
    echo "No pending records to load to MQTT."
fi

rm -f "$NEWRECS" "$FILE2"

echo "=== Process Data finished at $(date) ==="