Latest Contributions by AaronCole1

I do this kind of thing all the time! Love data mining. You have to group your requests in batches of whatever the max number of records that will be returned is, in this case 1000. Here's a code snippet I modified from something I use regularly.  You can set it up to run on task to be fully automated. You'll have to put in your desired output paths and it should run. import arcpy import requests import json import os import sys import traceback from datetime import date arcpy.env.overwriteOutput = True class DataScraper(): def __init__(self): # URL to map service you want to extract data from self.service_url = 'https://apps.deq.virginia.gov/arcgis/rest/services/public/EDMA/MapServer/104' def getServiceProperties(self, url): URL = url PARAMS = {'f' : 'json'} r = requests.get(url = URL, params = PARAMS) service_props = r.json() return service_props def getLayerIds(self, url, query=None): URL = url + '/query' PARAMS = {'f':'json', 'returnIdsOnly': True, 'where' : '1=1'} if query: PARAMS['where'] = "ST = '{}'".format(query) r = requests.get(url = URL, params = PARAMS) data = r.json() return data['objectIds'] def getLayerDataByIds(self, url, ids): # ids parameter should be a list of object ids URL = url + '/query' field = 'OBJECTID' value = ', '.join([str(i) for i in ids]) PARAMS = {'f': 'json', 'where': '{} IN ({})'.format(field, value), 'returnIdsOnly': False, 'returnCountOnly': False, 'returnGeometry': True, 'outFields': '*'} r = requests.post(url=URL, data=PARAMS) layer_data = r.json() return layer_data # UTILITIES def chunks(self, lst, n): # Yield successive n-sized chunks from list for i in range(0, len(lst), n): yield lst[i:i + n] if __name__ == '__main__': # Can use date for naming iterative runs... todays_date = date.today().strftime("%Y_%m_%d") # Instantiate DataScraper class ds = DataScraper() # Specify where you want to save output JSON and feature class on your machine out_json_path = rf"e:/data{todays_date}.json" out_fc_path = rf"e:/data{todays_date}.shp" # Function call to DataScraper class to get number of records, chunk up OIDs # and send requests to serivce. Results are compiled into single JSON and then # converted to shapefile/feature class at the end. def scrapeData(out_json_path, out_fc_path): try: service_props = ds.getServiceProperties(ds.service_url) max_record_count = service_props['maxRecordCount'] layer_ids = ds.getLayerIds(ds.service_url) id_groups = list(ds.chunks(layer_ids, max_record_count)) for i, id_group in enumerate(id_groups): print(' group {} of {}'.format(i+1, len(id_groups))) layer_data = ds.getLayerDataByIds(ds.service_url, id_group) if i==0: # If first iteration of id_groups layer_data_final = layer_data else: layer_data_final['features'].extend(layer_data['features']) print('Writing JSON file...') with open(out_json_path, 'w') as out_json_file: json.dump(layer_data_final, out_json_file) print('Converting JSON to features...') arcpy.conversion.JSONToFeatures(out_json_path, out_fc_path) except Exception: # Handle errors accordingly...this is generic tb = sys.exc_info()[2] tb_info = traceback.format_tb(tb)[0] pymsg = f'PYTHON ERRORS:\n\tTraceback info:\t{tb_info}\n\tError Info:\t{str(sys.exc_info()[1])}\n' msgs = f'ArcPy ERRORS:\t{arcpy.GetMessages(2)}\n' print(pymsg) print(msgs) # Run the function to get your data scrapeData(out_json_path, out_fc_path)   ... View more

Having a conversation with myself...but there is a setting in the Pro GP options that accomplishes this...thank goodness. Remove layers that reference data overwritten by geoprocessing tools ... View more