In [ ]:
import ee
import os
import shutil
from google.colab import drive
# ==============================================================================
# 1. AUTHENTICATION & GCP PROJECT INITIALIZATION
# ==============================================================================
PROJECT_ID = 'Redacted'
DISTRICT_NAME = 'Fatehgarh Sahib'
OUTPUT_FOLDER = 'PhD_Spatial_Mapping'
print("=" * 80)
print("LAYER 1: GCP PROJECT & GEE API INITIALIZATION CHECK")
print("=" * 80)
try:
ee.Initialize(project=PROJECT_ID)
print(f" GEE API initialized with GCP Project ID: '{PROJECT_ID}'")
except Exception as e:
print(f"⚠️ Standard initialization failed: {e}. Initiating authentication...")
ee.Authenticate()
ee.Initialize(project=PROJECT_ID)
print(f" Authenticated & initialized with GCP Project ID: '{PROJECT_ID}'")
# ==============================================================================
# 2. LOCAL ENVIRONMENT & GOOGLE DRIVE STORAGE AUDIT
# ==============================================================================
print("\n" + "=" * 80)
print("LAYER 2: GOOGLE DRIVE ACCESS & STORAGE CAPACITY AUDIT")
print("=" * 80)
drive.mount('/content/drive', force_remount=False)
drive_target_dir = f'/content/drive/MyDrive/{OUTPUT_FOLDER}'
if not os.path.exists(drive_target_dir):
os.makedirs(drive_target_dir, exist_ok=True)
print(f" Created missing destination directory: {drive_target_dir}")
else:
print(f" Destination directory verified: {drive_target_dir}")
total, used, free = shutil.disk_usage('/content/drive/MyDrive')
print(f" Drive Total Capacity : {total / (1024**3):.2f} GB")
print(f" Drive Used Space : {used / (1024**3):.2f} GB")
print(f" Drive Free Space : {free / (1024**3):.2f} GB")
if free / (1024**3) < 2.0:
print("⚠️ WARNING: Less than 2 GB free on Drive. Export might fail if storage fills up.")
else:
print(" Storage buffer check passed (>2 GB free).")
# ==============================================================================
# 3. SPATIAL GEOMETRY & BOUNDARY INTEGRITY
# ==============================================================================
print("\n" + "=" * 80)
print("LAYER 3: GAUL 2015 BOUNDARY & GRID SPECIFICATION")
print("=" * 80)
india_districts = ee.FeatureCollection("FAO/GAUL/2015/level2")
district_fc = india_districts.filter(ee.Filter.eq('ADM2_NAME', DISTRICT_NAME))
district_count = district_fc.size().getInfo()
if district_count == 0:
raise ValueError(f"CRITICAL: District '{DISTRICT_NAME}' not found in GAUL 2015!")
print(f" Boundary match confirmed: Found {district_count} feature for '{DISTRICT_NAME}'")
roi = district_fc.geometry()
roi_area_sqkm = roi.area().divide(1e6).getInfo()
bounds = roi.bounds().getInfo()['coordinates'][0]
min_lon, min_lat = bounds[0][0], bounds[0][1]
max_lon, max_lat = bounds[2][0], bounds[2][1]
print(f" District Area : {roi_area_sqkm:.2f} sq km")
print(f" Bounding Box (WGS84): Lon [{min_lon:.4f} to {max_lon:.4f}], Lat [{min_lat:.4f} to {max_lat:.4f}]")
# Approximate pixel grid calculation at 10m resolution (at ~30.5° N latitude)
lat_mid = (min_lat + max_lat) / 2.0
meters_per_deg_lat = 110852.0
meters_per_deg_lon = 111320.0 * (ee.Number(lat_mid).multiply(3.14159265/180).cos().getInfo())
width_meters = (max_lon - min_lon) * meters_per_deg_lon
height_meters = (max_lat - min_lat) * meters_per_deg_lat
grid_cols = int(width_meters / 10.0)
grid_rows = int(height_meters / 10.0)
total_raw_pixels = grid_cols * grid_rows
print(f" Estimated Grid Size: {grid_cols:,} cols x {grid_rows:,} rows (~{total_raw_pixels:,} total grid cells)")
# ==============================================================================
# 4. SATELLITE ASSETS & TIME-WINDOW AVAILABILITY CHECK
# ==============================================================================
print("\n" + "=" * 80)
print("LAYER 4: ASSET VERIFICATION & CLOUD COVERAGE AUDIT")
print("=" * 80)
# Check ESA WorldCover
worldcover = ee.ImageCollection("ESA/WorldCover/v200").first()
wc_bands = worldcover.bandNames().getInfo()
print(f" ESA WorldCover v200 accessible. Band present: {wc_bands}")
# Check Sentinel-2 collection availability across the 13 time windows
time_windows = [
('2021-10-15', '2021-10-31', 'NDVI_15-10'),
('2021-11-01', '2021-11-15', 'NDVI_01-11'),
('2021-11-16', '2021-11-30', 'NDVI_15-11'),
('2021-12-01', '2021-12-15', 'NDVI_01-12'),
('2021-12-16', '2021-12-31', 'NDVI_15-12'),
('2022-01-01', '2022-01-15', 'NDVI_01-01'),
('2022-01-16', '2022-01-31', 'NDVI_15-01'),
('2022-02-01', '2022-02-15', 'NDVI_01-02'),
('2022-02-16', '2022-02-28', 'NDVI_15-02'),
('2022-03-01', '2022-03-15', 'NDVI_01-03'),
('2022-03-16', '2022-03-31', 'NDVI_15-03'),
('2022-04-01', '2022-04-15', 'NDVI_01-04'),
('2022-04-16', '2022-04-30', 'NDVI_15-04')
]
print(f" Auditing Sentinel-2 Harmonized Granules over {DISTRICT_NAME} across 13 windows...")
total_scenes = 0
for start, end, label in time_windows:
count = ee.ImageCollection("COPERNICUS/S2_SR_HARMONIZED") \
.filterBounds(roi) \
.filterDate(start, end) \
.filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE', 20)) \
.size().getInfo()
total_scenes += count
status = "OK" if count > 0 else "EMPTY (Will fallback to dummy 0 band)"
print(f" - Window {label} [{start} to {end}]: {count} scenes found -> {status}")
print(f" Total clear/semi-clear scenes indexed: {total_scenes}")
# ==============================================================================
# 5. EXPORT METRICS & GEOTIFF FILE PROFILE
# ==============================================================================
print("\n" + "=" * 80)
print("LAYER 5: EXPECTED GEOTIFF FILE SPECIFICATIONS & SHARDING")
print("=" * 80)
num_bands = len(time_windows)
bytes_per_pixel = 4 # Float32 = 4 bytes
estimated_uncompressed_bytes = total_raw_pixels * num_bands * bytes_per_pixel
estimated_uncompressed_mb = estimated_uncompressed_bytes / (1024**2)
print(f" Number of Output Bands : {num_bands} (15-day median NDVI composites)")
print(f" Pixel Data Type : Float32 (32-bit floating point)")
print(f" Estimated Raw Uncompressed: ~{estimated_uncompressed_mb:.2f} MB")
print(f" Spatial Resolution : 10 meters / pixel")
print(f" Coordinate Reference Sys : EPSG:4326 (WGS84 Lat/Lon)")
# GEE sharding rule
if total_raw_pixels > 256000000:
print("⚠️ Sharding Note: Total pixel dimensions exceed standard single-tile limits.")
print(" GEE will automatically partition the export into multiple GeoTIFF files (e.g. -0000000000-0000000000.tif).")
else:
print(" Single-file export expected (Dimensions fit well within GEE's max tile threshold).")
print("=" * 80)
print(" ALL PRE-FLIGHT SANITY CHECKS COMPLETED SUCCESSFULLY.")
print("=" * 80)
================================================================================ LAYER 1: GCP PROJECT & GEE API INITIALIZATION CHECK ================================================================================ ⚠️ Standard initialization failed: Please authorize access to your Earth Engine account by running earthengine authenticate in your command line, or ee.Authenticate() in Python, and then retry.. Initiating authentication... Authenticated & initialized with GCP Project ID: 'redacted-project-id' ================================================================================ LAYER 2: GOOGLE DRIVE ACCESS & STORAGE CAPACITY AUDIT ================================================================================ Mounted at /content/drive Destination directory verified: /content/drive/MyDrive/PhD_Spatial_Mapping Drive Total Capacity : 107.72 GB Drive Used Space : 24.46 GB Drive Free Space : 83.26 GB Storage buffer check passed (>2 GB free). ================================================================================ LAYER 3: GAUL 2015 BOUNDARY & GRID SPECIFICATION ================================================================================ Boundary match confirmed: Found 1 feature for 'Fatehgarh Sahib' District Area : 1174.41 sq km Bounding Box (WGS84): Lon [76.0646 to 76.5812], Lat [30.4146 to 30.9153] Estimated Grid Size: 4,946 cols x 5,550 rows (~27,450,300 total grid cells) ================================================================================ LAYER 4: ASSET VERIFICATION & CLOUD COVERAGE AUDIT ================================================================================ ESA WorldCover v200 accessible. Band present: ['Map'] Auditing Sentinel-2 Harmonized Granules over Fatehgarh Sahib across 13 windows... - Window NDVI_15-10 [2021-10-15 to 2021-10-31]: 20 scenes found -> OK - Window NDVI_01-11 [2021-11-01 to 2021-11-15]: 20 scenes found -> OK - Window NDVI_15-11 [2021-11-16 to 2021-11-30]: 20 scenes found -> OK - Window NDVI_01-12 [2021-12-01 to 2021-12-15]: 15 scenes found -> OK - Window NDVI_15-12 [2021-12-16 to 2021-12-31]: 9 scenes found -> OK - Window NDVI_01-01 [2022-01-01 to 2022-01-15]: 0 scenes found -> EMPTY (Will fallback to dummy 0 band) - Window NDVI_15-01 [2022-01-16 to 2022-01-31]: 5 scenes found -> OK - Window NDVI_01-02 [2022-02-01 to 2022-02-15]: 14 scenes found -> OK - Window NDVI_15-02 [2022-02-16 to 2022-02-28]: 12 scenes found -> OK - Window NDVI_01-03 [2022-03-01 to 2022-03-15]: 12 scenes found -> OK - Window NDVI_15-03 [2022-03-16 to 2022-03-31]: 24 scenes found -> OK - Window NDVI_01-04 [2022-04-01 to 2022-04-15]: 20 scenes found -> OK - Window NDVI_15-04 [2022-04-16 to 2022-04-30]: 17 scenes found -> OK Total clear/semi-clear scenes indexed: 188 ================================================================================ LAYER 5: EXPECTED GEOTIFF FILE SPECIFICATIONS & SHARDING ================================================================================ Number of Output Bands : 13 (15-day median NDVI composites) Pixel Data Type : Float32 (32-bit floating point) Estimated Raw Uncompressed: ~1361.29 MB Spatial Resolution : 10 meters / pixel Coordinate Reference Sys : EPSG:4326 (WGS84 Lat/Lon) Single-file export expected (Dimensions fit well within GEE's max tile threshold). ================================================================================ ALL PRE-FLIGHT SANITY CHECKS COMPLETED SUCCESSFULLY. ================================================================================
In [ ]:
import ee
from google.colab import drive
# ==========================================
# 1. SETUP & AUTHENTICATION
# ==========================================
print("Mounting Google Drive...")
drive.mount('/content/drive', force_remount=True)
print("Authenticating Earth Engine...")
try:
ee.Initialize(project='redacted') #
except Exception as e:
ee.Authenticate()
ee.Initialize(project='redacted')
# ==========================================
# 2. CONFIGURATION
# ==========================================
DISTRICT_NAME = 'Fatehgarh Sahib'
OUTPUT_FOLDER = 'PhD_Spatial_Mapping'
OUTPUT_FILENAME = f'{DISTRICT_NAME.replace(" ", "_")}_Rabi_13Band_NDVI_2021_2022'
print(f"\nInitializing District-Wide Spatial Extraction for {DISTRICT_NAME}...")
india_districts = ee.FeatureCollection("FAO/GAUL/2015/level2")
roi = india_districts.filter(ee.Filter.eq('ADM2_NAME', DISTRICT_NAME)).geometry()
print("ROI area (sq km):", roi.area().divide(1e6).getInfo())
worldcover = ee.ImageCollection("ESA/WorldCover/v200").first()
ag_mask = worldcover.eq(40)
print(" ROI and ESA WorldCover Cropland Mask loaded.")
# ==========================================
# 4. SATELLITE PROCESSING FUNCTIONS
# (identical to the validated Patiala pipeline -- unchanged)
# ==========================================
def maskS2clouds(image):
qa = image.select('QA60')
cloudBitMask = 1 << 10
cirrusBitMask = 1 << 11
mask = qa.bitwiseAnd(cloudBitMask).eq(0).And(qa.bitwiseAnd(cirrusBitMask).eq(0))
return image.updateMask(mask).copyProperties(image, ["system:time_start"])
def add_ndvi(image):
ndvi = image.normalizedDifference(['B8', 'B4']).rename('NDVI').toFloat()
return image.addBands(ndvi)
# ==========================================
# 5. DEFINE 13 x 15-DAY TIME WINDOWS
# (identical dates to the Patiala extraction -- same Rabi season)
# ==========================================
time_windows = [
('2021-10-15', '2021-10-31', 'NDVI_15-10'),
('2021-11-01', '2021-11-15', 'NDVI_01-11'),
('2021-11-16', '2021-11-30', 'NDVI_15-11'),
('2021-12-01', '2021-12-15', 'NDVI_01-12'),
('2021-12-16', '2021-12-31', 'NDVI_15-12'),
('2022-01-01', '2022-01-15', 'NDVI_01-01'),
('2022-01-16', '2022-01-31', 'NDVI_15-01'),
('2022-02-01', '2022-02-15', 'NDVI_01-02'),
('2022-02-16', '2022-02-28', 'NDVI_15-02'),
('2022-03-01', '2022-03-15', 'NDVI_01-03'),
('2022-03-16', '2022-03-31', 'NDVI_15-03'),
('2022-04-01', '2022-04-15', 'NDVI_01-04'),
('2022-04-16', '2022-04-30', 'NDVI_15-04')
]
# ==========================================
# 6. SPATIAL STACKING (with the winter-fog empty-collection guard)
# ==========================================
print("\nConnecting to Sentinel-2 and generating 15-day median composites...")
dummy_image = ee.Image.constant(0).toFloat().rename('NDVI').updateMask(0)
bands = []
for start, end, name in time_windows:
collection = ee.ImageCollection("COPERNICUS/S2_SR_HARMONIZED") \
.filterBounds(roi) \
.filterDate(start, end) \
.filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE', 20)) \
.map(maskS2clouds) \
.map(add_ndvi) \
.select('NDVI')
window_img = collection.merge(ee.ImageCollection([dummy_image])) \
.median() \
.rename(name)
bands.append(window_img)
multi_band_image = ee.ImageCollection(bands).toBands()
clean_band_names = [window[2] for window in time_windows]
multi_band_image = multi_band_image.rename(clean_band_names)
final_masked_image = multi_band_image.toFloat().updateMask(ag_mask).clip(roi)
# ==========================================
# 7. EXPORT TO DRIVE
# ==========================================
print(f"\nSubmitting task to Google Earth Engine to export {OUTPUT_FILENAME}.tif...")
task = ee.batch.Export.image.toDrive(
image=final_masked_image,
description=OUTPUT_FILENAME,
folder=OUTPUT_FOLDER,
fileNamePrefix=OUTPUT_FILENAME,
region=roi,
scale=10,
crs='EPSG:4326',
maxPixels=1e13,
fileFormat='GeoTIFF'
)
task.start()
print(f"\nExport task submitted: {OUTPUT_FILENAME}.tif")
Mounting Google Drive... Mounted at /content/drive Authenticating Earth Engine... Initializing District-Wide Spatial Extraction for Fatehgarh Sahib... ROI area (sq km): 1174.4102610014593 ROI and ESA WorldCover Cropland Mask loaded. Connecting to Sentinel-2 and generating 15-day median composites... Submitting task to Google Earth Engine to export Fatehgarh_Sahib_Rabi_13Band_NDVI_2021_2022.tif... Export task submitted: Fatehgarh_Sahib_Rabi_13Band_NDVI_2021_2022.tif
In [ ]:
import rasterio
from rasterio.windows import Window
import numpy as np
import pandas as pd
from scipy.signal import savgol_filter
import os
import gc
import joblib
from google.colab import drive
import warnings
warnings.filterwarnings('ignore')
# ==========================================
# 1. SETUP & CONFIGURATION
# ==========================================
print("Mounting Google Drive...")
drive.mount('/content/drive', force_remount=True)
FOLDER_PATH = '/content/drive/MyDrive/PhD_Spatial_Mapping/'
EXACT_FILENAME = 'Fatehgarh_Sahib_Rabi_13Band_NDVI_2021_2022.tif'
tile_files = [os.path.join(FOLDER_PATH, EXACT_FILENAME)]
# FROZEN MODEL PATHS -- loaded, never refit, never re-derived
MODEL_PATH = '/content/drive/MyDrive/PhD_Spatial_Mapping/Models/xgboost_ndvi_only.pkl'
ENCODER_PATH = '/content/drive/MyDrive/PhD_Spatial_Mapping/Models/label_encoder.pkl'
print("\n============================================================")
print(" INITIATING SPATIAL GENERALIZATION INFERENCE (Fatehgarh Sahib)")
print("============================================================")
print(f" Target GeoTIFF File : {EXACT_FILENAME}")
print(f" Target Model Path : {MODEL_PATH}")
print(f" Target Encoder Path : {ENCODER_PATH}")
if not os.path.exists(tile_files[0]):
raise FileNotFoundError(f"CRITICAL: Could not find {EXACT_FILENAME} in {FOLDER_PATH}. Verify GEE export finished.")
# ==========================================
# 2. LOAD THE FROZEN PATIALA MODEL
# ==========================================
print("\nLoading frozen XGBoost model and label encoder (Patiala-trained, unmodified)...")
model_xgb = joblib.load(MODEL_PATH)
encoder = joblib.load(ENCODER_PATH)
print(f"Encoder classes detected: {list(encoder.classes_)}")
class_pixel_counts = {}
BLOCK_SIZE = 2000 # Memory-safe chunk size
# ==========================================
# 3. CHUNKED TILE PROCESSING & INFERENCE
# ==========================================
for i, fp in enumerate(tile_files):
file_name = os.path.basename(fp)
out_fp = fp.replace('.tif', '_CLASSIFIED.tif')
print(f"\nProcessing Tile {i+1}/{len(tile_files)}: {file_name}")
with rasterio.open(fp) as src:
meta = src.meta.copy()
n_bands = src.count
height = src.height
width = src.width
meta.update({'count': 1, 'dtype': 'int16', 'nodata': 0})
with rasterio.open(out_fp, 'w', **meta) as dst:
for row_idx in range(0, height, BLOCK_SIZE):
for col_idx in range(0, width, BLOCK_SIZE):
window_height = min(BLOCK_SIZE, height - row_idx)
window_width = min(BLOCK_SIZE, width - col_idx)
window = Window(col_idx, row_idx, window_width, window_height)
data = src.read(window=window)
flat_data = data.reshape(n_bands, -1).T
# 0.0 represents masked non-crop / missing data
flat_data[flat_data == 0.0] = np.nan
miss_pct = np.isnan(flat_data).mean(axis=1)
valid_mask = miss_pct <= 0.8
valid_pixels = flat_data[valid_mask]
chunk_predictions = np.zeros(window_height * window_width, dtype=np.int16)
if len(valid_pixels) > 0:
# Spike removal threshold (|diff| > 0.4)
diffs = np.abs(np.diff(valid_pixels, axis=1))
spike_locations = np.hstack(
(np.zeros((len(valid_pixels), 1), dtype=bool), diffs > 0.4)
)
valid_pixels[spike_locations] = np.nan
df_pixels = pd.DataFrame(valid_pixels)
df_pixels = df_pixels.interpolate(
method='linear', axis=1, limit_direction='both'
).fillna(0)
smoothed_pixels = df_pixels.values
# Savitzky-Golay filter (window=5, polyorder=2)
smoothed_pixels = savgol_filter(
smoothed_pixels, window_length=5, polyorder=2, axis=1
)
smoothed_pixels = np.clip(smoothed_pixels, -1.0, 1.0)
smoothed_pixels = smoothed_pixels.astype(np.float32)
# --- INFERENCE: Frozen model, .predict() only ---
encoded_preds = model_xgb.predict(smoothed_pixels)
chunk_predictions[valid_mask] = encoded_preds + 1
unique_labels, counts = np.unique(encoded_preds, return_counts=True)
for u_label, count in zip(unique_labels, counts):
class_pixel_counts[u_label] = class_pixel_counts.get(u_label, 0) + count
chunk_2d = chunk_predictions.reshape(1, window_height, window_width)
dst.write(chunk_2d, window=window)
del data, flat_data, valid_mask, chunk_predictions
gc.collect()
# ==========================================
# 4. RESULTS AGGREGATION & REPORTING
# ==========================================
print("\n" + "="*50)
print("INFERENCE COMPLETE: PER-CLASS PIXEL BREAKDOWN")
print("="*50)
for enc_label, pixel_count in sorted(class_pixel_counts.items(), key=lambda x: x[1], reverse=True):
class_name = encoder.classes_[enc_label]
print(f" [Label Index {enc_label}] {class_name}: {pixel_count:,} pixels")
HECTARES_PER_PIXEL = 0.01 # 10m x 10m = 100 sq m = 0.01 hectares
# Automatically sum labels containing 'wheat' (case-insensitive safeguard)
wheat_encoded_labels = [
i for i, label in enumerate(encoder.classes_)
if 'wheat' in str(label).lower()
]
total_wheat_pixels = sum(
count for enc_label, count in class_pixel_counts.items() if enc_label in wheat_encoded_labels
)
ai_predicted_wheat_ha = total_wheat_pixels * HECTARES_PER_PIXEL
print("="*50)
print(f"Total Identified Wheat Pixels : {total_wheat_pixels:,}")
print(f"AI Predicted Wheat Area : {ai_predicted_wheat_ha:,.2f} Ha")
print("="*50)
Mounting Google Drive... Mounted at /content/drive ============================================================ INITIATING SPATIAL GENERALIZATION INFERENCE (Fatehgarh Sahib) ============================================================ Target GeoTIFF File : Fatehgarh_Sahib_Rabi_13Band_NDVI_2021_2022.tif Target Model Path : /content/drive/MyDrive/PhD_Spatial_Mapping/Models/xgboost_ndvi_only.pkl Target Encoder Path : /content/drive/MyDrive/PhD_Spatial_Mapping/Models/label_encoder.pkl Loading frozen XGBoost model and label encoder (Patiala-trained, unmodified)... Encoder classes detected: ['Barren (Combined)', 'Fodder / Berseem', 'Forest (Combined)', 'Forest / Tree Cover', 'Mustard', 'Potato & Short Rabi', 'Urban (Combined)', 'Water', 'Wheat (Late / Double)', 'Wheat (Late / Double) (Atmospheric Artifact)', 'Wheat (Standard) (Atmospheric Artifact)', 'Wheat (Standard) (Combined)'] Processing Tile 1/1: Fatehgarh_Sahib_Rabi_13Band_NDVI_2021_2022.tif ================================================== INFERENCE COMPLETE: PER-CLASS PIXEL BREAKDOWN ================================================== [Label Index 11] Wheat (Standard) (Combined): 8,139,048 pixels [Label Index 5] Potato & Short Rabi: 2,379,129 pixels [Label Index 8] Wheat (Late / Double): 458,217 pixels [Label Index 4] Mustard: 295,241 pixels [Label Index 2] Forest (Combined): 242,726 pixels [Label Index 1] Fodder / Berseem: 204,463 pixels [Label Index 0] Barren (Combined): 116,317 pixels [Label Index 6] Urban (Combined): 83,180 pixels [Label Index 10] Wheat (Standard) (Atmospheric Artifact): 71,540 pixels [Label Index 3] Forest / Tree Cover: 67,405 pixels [Label Index 9] Wheat (Late / Double) (Atmospheric Artifact): 53,436 pixels [Label Index 7] Water: 1,055 pixels ================================================== Total Identified Wheat Pixels : 8,722,241 AI Predicted Wheat Area : 87,222.41 Ha ==================================================
In [ ]: