In [ ]:

output_dir = Path('output')
output_dir.mkdir(exist_ok=True)

# Write a small example output for demonstration purposes
sample_output = output_dir / 'sample_gcp_output.xlsx'
df_processed[[name_col, 'Latitude', 'Longitude', 'Latitude_DMS', 'Longitude_DMS', 'Remarks']].to_excel(sample_output, index=False)

print(f"Example output saved to: {sample_output}")

output_dir = Path('output') output_dir.mkdir(exist_ok=True) # Write a small example output for demonstration purposes sample_output = output_dir / 'sample_gcp_output.xlsx' df_processed[[name_col, 'Latitude', 'Longitude', 'Latitude_DMS', 'Longitude_DMS', 'Remarks']].to_excel(sample_output, index=False) print(f"Example output saved to: {sample_output}")

Notebook 1 — Save Outputs¶

The script writes shapefiles and Excel outputs to the selected directories after validation.

In [ ]:

df_processed['Latitude_DMS'] = df_processed['Latitude'].apply(lambda x: dms_from_decimal(x, 'lat'))
df_processed['Longitude_DMS'] = df_processed['Longitude'].apply(lambda x: dms_from_decimal(x, 'lon'))

df_processed[['Name' if 'Name' in df_processed.columns else name_col, 'Latitude', 'Longitude', 'Latitude_DMS', 'Longitude_DMS']].head()

df_processed['Latitude_DMS'] = df_processed['Latitude'].apply(lambda x: dms_from_decimal(x, 'lat')) df_processed['Longitude_DMS'] = df_processed['Longitude'].apply(lambda x: dms_from_decimal(x, 'lon')) df_processed[['Name' if 'Name' in df_processed.columns else name_col, 'Latitude', 'Longitude', 'Latitude_DMS', 'Longitude_DMS']].head()

Notebook 1 — Build the Processed DataFrame with Derived Fields¶

This section adds DMS coordinate strings and preserves the original information needed for downstream reporting.

In [ ]:

df_processed['Remarks'] = df_processed[name_col].apply(classify_gcp)
df_processed['Remarks'].value_counts()

df_processed['Remarks'] = df_processed[name_col].apply(classify_gcp) df_processed['Remarks'].value_counts()

Notebook 1 — Classify GCP Names as Temporary or Permanent¶

The naming convention is used to identify temporary survey points and permanent control points, which affects the output folders and reporting.

In [ ]:

df_processed = df.copy()
df_processed['Latitude'] = None
df_processed['Longitude'] = None

for idx, row in df_processed.iterrows():
    easting = float(row[east_col])
    northing = float(row[north_col])
    lat, lon = utm_to_latlon(easting, northing, UTM_EPSG)
    df_processed.at[idx, 'Latitude'] = round(lat, 8)
    df_processed.at[idx, 'Longitude'] = round(lon, 8)

print(df_processed[['Name' if 'Name' in df_processed.columns else name_col, 'Latitude', 'Longitude']].head())

df_processed = df.copy() df_processed['Latitude'] = None df_processed['Longitude'] = None for idx, row in df_processed.iterrows(): easting = float(row[east_col]) northing = float(row[north_col]) lat, lon = utm_to_latlon(easting, northing, UTM_EPSG) df_processed.at[idx, 'Latitude'] = round(lat, 8) df_processed.at[idx, 'Longitude'] = round(lon, 8) print(df_processed[['Name' if 'Name' in df_processed.columns else name_col, 'Latitude', 'Longitude']].head())

Notebook 1 — Convert UTM Coordinates to Latitude/Longitude¶

This section converts projected coordinates into geographic coordinates using the chosen EPSG code.

In [ ]:

required_aliases = {
    'name': ['Name', 'name', 'GCP Name', 'Point Name'],
    'easting': ['Easting', 'Easting (m)', 'E', 'X'],
    'northing': ['Northing', 'Northing (m)', 'N', 'Y'],
}


def find_matching_column(df, aliases):
    for alias in aliases:
        if alias in df.columns:
            return alias
    return None


name_col = find_matching_column(df, required_aliases['name'])
east_col = find_matching_column(df, required_aliases['easting'])
north_col = find_matching_column(df, required_aliases['northing'])

if name_col is None or east_col is None or north_col is None:
    raise ValueError(
        f"Missing required columns. Found: {list(df.columns)}"
    )

print(f"Using columns -> Name: {name_col}, Easting: {east_col}, Northing: {north_col}")

required_aliases = { 'name': ['Name', 'name', 'GCP Name', 'Point Name'], 'easting': ['Easting', 'Easting (m)', 'E', 'X'], 'northing': ['Northing', 'Northing (m)', 'N', 'Y'], } def find_matching_column(df, aliases): for alias in aliases: if alias in df.columns: return alias return None name_col = find_matching_column(df, required_aliases['name']) east_col = find_matching_column(df, required_aliases['easting']) north_col = find_matching_column(df, required_aliases['northing']) if name_col is None or east_col is None or north_col is None: raise ValueError( f"Missing required columns. Found: {list(df.columns)}" ) print(f"Using columns -> Name: {name_col}, Easting: {east_col}, Northing: {north_col}")

Notebook 1 — Validate Required Fields and Handle Missing Data¶

This section checks whether the expected fields are present and makes the failure mode explicit if the input table is incomplete.

In [ ]:

from pathlib import Path


def read_input(file_path):
    file_path = Path(file_path)
    ext = file_path.suffix.lower()

    if ext in {".xlsx", ".xls"}:
        df = pd.read_excel(file_path)
    elif ext == ".csv":
        df = pd.read_csv(file_path)
    else:
        raise ValueError("Unsupported file format. Use .xlsx, .xls, or .csv")

    df.columns = [str(col).strip() for col in df.columns]
    print(f"Loaded {len(df)} rows from {file_path.name}")
    print(df.head())
    return df


df = read_input(INPUT_FILE)

from pathlib import Path def read_input(file_path): file_path = Path(file_path) ext = file_path.suffix.lower() if ext in {".xlsx", ".xls"}: df = pd.read_excel(file_path) elif ext == ".csv": df = pd.read_csv(file_path) else: raise ValueError("Unsupported file format. Use .xlsx, .xls, or .csv") df.columns = [str(col).strip() for col in df.columns] print(f"Loaded {len(df)} rows from {file_path.name}") print(df.head()) return df df = read_input(INPUT_FILE)

Notebook 1 — Read Excel/CSV and Normalize Columns¶

The next step is to inspect the source data and standardize the column names so the script can reliably detect the required fields.

Notebook 1 — Import Libraries and Define Constants¶

This section imports the core libraries for reading, transforming, and writing geospatial outputs. The constants below help define the input file, chosen UTM CRS, and output folder used in the workflow.

In [ ]:

import os
import math
import pandas as pd
from pyproj import Transformer
import shapefile

# Configuration
INPUT_FILE = r"C:/path/to/your/gcp_input.xlsx"
UTM_EPSG = 32644
OUTPUT_DIR = r"C:/path/to/output_folder"

# Helper functions

def utm_to_latlon(easting, northing, epsg):
    transformer = Transformer.from_crs(f"EPSG:{epsg}", "EPSG:4326", always_xy=True)
    lon, lat = transformer.transform(easting, northing)
    return lat, lon


def dd_to_dms(decimal_degrees):
    is_negative = decimal_degrees < 0
    decimal_degrees = abs(decimal_degrees)
    degrees = int(decimal_degrees)
    minutes_float = (decimal_degrees - degrees) * 60
    minutes = int(minutes_float)
    seconds = (minutes_float - minutes) * 60
    return degrees, minutes, seconds, is_negative


def format_dms_lat(lat):
    d, m, s, is_neg = dd_to_dms(lat)
    direction = "S" if is_neg else "N"
    return f"{d}°{m:02d}'{s:06.3f}\"{direction}"


def format_dms_lon(lon):
    d, m, s, is_neg = dd_to_dms(lon)
    direction = "W" if is_neg else "E"
    return f"{d}°{m:02d}'{s:06.3f}\"{direction}"


def classify_gcp(name):
    return "Temporary GCP" if str(name).strip().upper().startswith("GCP") else "Permanent GCP"

import os import math import pandas as pd from pyproj import Transformer import shapefile # Configuration INPUT_FILE = r"C:/path/to/your/gcp_input.xlsx" UTM_EPSG = 32644 OUTPUT_DIR = r"C:/path/to/output_folder" # Helper functions def utm_to_latlon(easting, northing, epsg): transformer = Transformer.from_crs(f"EPSG:{epsg}", "EPSG:4326", always_xy=True) lon, lat = transformer.transform(easting, northing) return lat, lon def dd_to_dms(decimal_degrees): is_negative = decimal_degrees < 0 decimal_degrees = abs(decimal_degrees) degrees = int(decimal_degrees) minutes_float = (decimal_degrees - degrees) * 60 minutes = int(minutes_float) seconds = (minutes_float - minutes) * 60 return degrees, minutes, seconds, is_negative def format_dms_lat(lat): d, m, s, is_neg = dd_to_dms(lat) direction = "S" if is_neg else "N" return f"{d}°{m:02d}'{s:06.3f}\"{direction}" def format_dms_lon(lon): d, m, s, is_neg = dd_to_dms(lon) direction = "W" if is_neg else "E" return f"{d}°{m:02d}'{s:06.3f}\"{direction}" def classify_gcp(name): return "Temporary GCP" if str(name).strip().upper().startswith("GCP") else "Permanent GCP"

GCP / PGCP Processing Workflow¶

This notebook documents the end-to-end workflow for converting GCP/PGCP records into georeferenced outputs, including shapefiles and an Excel summary workbook.

The companion script is available at docs/assets/GCPs_IBM_deliverables_V1.py.