I'm busy rewriting my hydrological model python scripts into functions that are modular and found that my original python script runs twice as fast as my updated python script. The only reason I can find is that the original scripts outputs are saved in memory before being passed onto the next function. My new python script is writing and reading each output from disk. I might be incorrect in my understanding. If anyone can give me guidance on improving my new python script in both pythonic structure and performance.
Original Python Script
'''
Created on May 20, 2015
@author: PeterW
'''
# import system modules and site packages
import os
import time
import arcpy
import ArcHydroTools
# check out Spatial Analyst Extension
arcpy.CheckOutExtension("Spatial")
# set environment settings
arcpy.env.overwriteOutput = True
# set input and output arguments
raw = r"E:\Python\Temp\Model04\DEM\raw"
rasWs = r"E:\Python\Temp\Model04\Layers04"
outWs = r"E:\Python\Temp\Model04\Model04.gdb"
# Processing time
def hms_string(sec_elapsed):
h = int(sec_elapsed / (60 * 60))
m = int(sec_elapsed % (60 * 60) / 60)
s = sec_elapsed % 60
return "{}h:{:>02}m:{:>05.2f}s".format(h, m, s)
start_time1 = time.time()
# archydro variables
fill_sinks = os.path.join(rasWs, "fil")
flow_dir = os.path.join(rasWs, "fdr")
flow_acc = os.path.join(rasWs, "fac")
streams = os.path.join(rasWs, "str")
stream_seg = os.path.join(rasWs, "strlnk")
catchment_grid = os.path.join(rasWs, "cat")
catchment_poly = os.path.join(outWs, "Layers","Catchment")
drainage_line = os.path.join(outWs, "Layers", "DrainageLine")
adj_catch = os.path.join(outWs, "Layers", "AdjointCatchment")
try:
# calculate the fill sinks
arcpy.AddMessage("Processing Fill Sinks")
ArcHydroTools.FillSinks(raw, fill_sinks)
# calculate the flow direction
arcpy.AddMessage("Processing Flow Direction")
ArcHydroTools.FlowDirection(fill_sinks, flow_dir)
# calculate the flow accumulation
arcpy.AddMessage("Processing Flow Accumulation")
ArcHydroTools.FlowAccumulation(flow_dir, flow_acc)
# calculate the maximum flow accumulation
arcpy.AddMessage("Processing Flow Accumulation Maximum")
maxcellsResult = arcpy.GetRasterProperties_management(flow_acc, "MAXIMUM")
maxcells = maxcellsResult.getOutput(0)
print maxcells
# calculate the stream threshold number of cells
arcpy.AddMessage("Processing Stream Threshold")
stream_threshold_numcells = (int(maxcells)*0.125/100)
print stream_threshold_numcells
# calculate the stream definition
arcpy.AddMessage("Processing Stream Definition")
ArcHydroTools.StreamDefinition(flow_acc, stream_threshold_numcells, streams)
# calculate the stream segmentation
arcpy.AddMessage("Processing Stream Segmentation")
ArcHydroTools.StreamSegmentation(streams, flow_dir, stream_seg)
# calculate the catchment grid delineation
arcpy.AddMessage("Processing Catchment Grid Delineation")
ArcHydroTools.CatchmentGridDelineation(flow_dir, stream_seg, catchment_grid)
# calculate the catchment polygons from the catchment grid
arcpy.AddMessage("Processing Catchment Polygons")
ArcHydroTools.CatchmentPolyProcessing(catchment_grid, catchment_poly)
# calculate the drainage lines from the stream segmentation grid
arcpy.AddMessage("Processing DrainageLines")
ArcHydroTools.DrainageLineProcessing(stream_seg, flow_dir, drainage_line)
# calculate the adjoint catchment polygons
arcpy.AddMessage("Processing Ajdoint Catchments")
ArcHydroTools.AdjointCatchment(drainage_line, catchment_poly, adj_catch)
arcpy.AddMessage("Completed Processing archydro Main Model")
except:
print(arcpy.GetMessages(2))
pass
# Determine the time take to process hydrological characteristics
end_time1 = time.time()
print ("It took {} to process hydrological characteristics".format(hms_string(end_time1 - start_time1)))
arcpy.CheckInExtension("Spatial")
New Python Script
'''
Created on Feb 24, 2016
@author: PeterW
'''
# import system modules and site packages
import time
from pathlib import Path
import arcpy
import ArcHydroTools
# check out extension
arcpy.CheckOutExtension("Spatial")
# set environment settings
arcpy.env.overwriteOutput = True
# set input and output arguments
dem = r"E:\Python\Temp\Model04\DEM\raw"
raster_workspace = r"E:\Python\Temp\Model04\Layers04"
fgdb = Path(r"E:\Python\Temp\Model04\Model04.gdb")
# Processing time
def hms_string(sec_elapsed):
h = int(sec_elapsed / (60 * 60))
m = int(sec_elapsed % (60 * 60) / 60)
s = sec_elapsed % 60
return "{}h:{:>02}m:{:>05.2f}s".format(h, m, s)
start_time1 = time.time()
# generate fill sinks grid
def fill_sinks(dem, raster_workspace):
fill_sinks = "{0}\\{1}".format(raster_workspace, "fil")
arcpy.AddMessage("Processing Fill Sinks")
ArcHydroTools.FillSinks(dem, fill_sinks)
fill_sinks(dem, raster_workspace)
# generate flow direction grid
def flow_direction(raster_workspace):
fill_sinks = "{0}\\{1}".format(raster_workspace, "fil")
flow_direction = "{0}\\{1}".format(raster_workspace, "fdr")
arcpy.AddMessage("Processing Flow Direction")
ArcHydroTools.FlowDirection(fill_sinks, flow_direction)
flow_direction(raster_workspace)
# generate flow accumulation grid
def flow_accumulation(raster_workspace):
flow_direction = "{0}\\{1}".format(raster_workspace, "fdr")
flow_accumulation = "{0}\\{1}".format(raster_workspace, "fac")
arcpy.AddMessage("Processing Flow Accumulation")
ArcHydroTools.FlowAccumulation(flow_direction, flow_accumulation)
flow_accumulation(raster_workspace)
# calculate stream threshold based on 0.5% of maximum flow accumulation
def stream_threshold(raster_workspace):
flow_accumulation = "{0}\\{1}".format(raster_workspace, "fac")
arcpy.AddMessage("Processing Flow Stream Threshold")
maxcellsResult = arcpy.GetRasterProperties_management(flow_accumulation, "MAXIMUM")
maxcells = maxcellsResult.getOutput(0)
arcpy.AddMessage("{} Maximum Cells".format(maxcells))
stream_threshold_numcells = (int(maxcells)*0.5/100)
arcpy.AddMessage("{} Stream Threshold".format(stream_threshold_numcells))
return stream_threshold_numcells
numcells = stream_threshold(raster_workspace)
# generate the stream definition grid
def stream_definition(raster_workspace):
flow_accumulation = "{0}\\{1}".format(raster_workspace, "fac")
stream = "{0}\\{1}".format(raster_workspace, "str")
arcpy.AddMessage("Processing Stream Definition")
ArcHydroTools.StreamDefinition(flow_accumulation, numcells, stream)
stream_definition(raster_workspace)
# generate the stream segmentation grid
def stream_segmentation(raster_workspace):
stream = "{0}\\{1}".format(raster_workspace, "str")
flow_direction = "{0}\\{1}".format(raster_workspace, "fac")
stream_link = "{0}\\{1}".format(raster_workspace, "strlnk")
arcpy.AddMessage("Processing Stream Segmentation")
ArcHydroTools.StreamSegmentation(stream, flow_direction, stream_link)
stream_segmentation(raster_workspace)
# calculate the catchment grid delineation
def catchment_grid(raster_workspace):
flow_direction = "{0}\\{1}".format(raster_workspace, "fdr")
stream_link = "{0}\\{1}".format(raster_workspace, "strlnk")
catchment_grid = "{0}\\{1}".format(raster_workspace, "cat")
arcpy.AddMessage("Processing Catchment Grid Delineation")
ArcHydroTools.CatchmentGridDelineation(flow_direction, stream_link, catchment_grid)
catchment_grid(raster_workspace)
# calculate the catchment polygons from the catchment grid
def catchment_polygon(raster_workspace, fgdb):
catchment_grid = "{0}\\{1}".format(raster_workspace, "cat")
catchment_polygon = "{0}\\{1}".format(Path(fgdb, "Layers"), "Catchment")
arcpy.AddMessage("Processing Catchment Polygon")
ArcHydroTools.CatchmentPolyProcessing(catchment_grid, catchment_polygon)
catchment_polygon(raster_workspace, fgdb)
# calculate the drainage lines from the stream segmentation grid
def drainage_line(raster_workspace, fgdb):
stream_link = "{0}\\{1}".format(raster_workspace, "strlnk")
flow_direction = "{0}\\{1}".format(raster_workspace, "fdr")
drainage_line = "{0}\\{1}".format(Path(fgdb, "Layers"), "DrainageLine")
arcpy.AddMessage("Processing DrainageLine")
ArcHydroTools.DrainageLineProcessing(stream_link, flow_direction, drainage_line)
drainage_line(raster_workspace, fgdb)
# calculate the adjoint catchment polygons
def adjoint_catchment(fgdb):
drainage_line = "{0}\\{1}".format(Path(fgdb, "Layers"), "DrainageLine")
catchment_polygon = "{0}\\{1}".format(Path(fgdb, "Layers"), "Catchment")
adjoint_catchment = "{0}\\{1}".format(Path(fgdb, "Layers"), "AdjointCatchment")
arcpy.AddMessage("Processing Adjoint Catchment")
ArcHydroTools.AdjointCatchment(drainage_line, catchment_polygon, adjoint_catchment)
adjoint_catchment(fgdb)
arcpy.CheckInExtension("Spatial")
# Determine the time take to process hydrological characteristics
end_time1 = time.time()
print ("It took {} to process hydrological characteristics".format(hms_string(end_time1 - start_time1)))