Copy Feature Service from AGOL TO FGDB

AndrewCallison · ‎08-04-2015

import json
import urllib
import arcpy
import os
  
USERNAME = "  "  
PASSWORD = "  "  
REFER = "http://www.arcgis.com"  
  
def urlopen(url, data=None):  
    referer = "http://arcgis.com/arcgis/rest"  
    req = urllib.Request(url)  
    req.add_header('Referer', referer)  
  
    if data:  
        response = urllib.urlopen(req, data)  
    else:  
        response = urllib.urlopen(req)  
  
    return response  
  
  
def gentoken(username, password, referer, expiration=60):  
    #Re-usable function to get a token     
      
    query_dict = {'username': username,  
                  'password': password,  
                  'expiration': str(expiration),  
                  'client': 'referer',  
                  'referer': referer,  
                  'f': 'json'}      
      
    query_string = urllib.urlencode(query_dict)  
    tokenUrl = "https://www.arcgis.com/sharing/rest/generateToken"  
      
    tokenResponse = urllib.urlopen(tokenUrl, urllib.urlencode(query_dict))     
    token = json.loads(tokenResponse.read())  
          
    if "token" not in token:  
        print (token['messages'])  
        exit()  
    else:  
        # Return the token to the function which called for it  
        return token['token']  
      
# Modify the following variables:
# URL to your service, where clause, fields and token if applicable
baseURL= "http://services.arcgis.com/ /arcgis/rest/services/MappingLicenses/FeatureServer/0/query"
where = '1=1'
fields = '**
token = gentoken(USERNAME, PASSWORD, REFER)
query = "?where={}&outFields={}&returnGeometry=true&f=json&token={}".format(where, fields, token)
fsURL = baseURL +query
fs = arcpy.FeatureSet()
fs.load(fsURL)



arcpy.CopyFeatures_management(fs, r"F:\Data Source\Copy_AGOL.gdb\Test")

Getting this Error:

Python 3.4.1 (v3.4.1:c0e311e010fc, May 18 2014, 10:45:13) [MSC v.1600 64 bit (AMD64)] on win32

Type "copyright", "credits" or "license()" for more information.

>>> ================================ RESTART ================================

>>>

Traceback (most recent call last):

File "C:\Users\agcallis\Desktop\CopyFS.py", line 51, in <module>

token = gentoken(USERNAME, PASSWORD, REFER)

File "C:\Users\agcallis\Desktop\CopyFS.py", line 33, in gentoken

query_string = urllib.urlencode(query_dict)

AttributeError: 'module' object has no attribute 'urlencode'

What am I do wrong??

Thanks

IanMurray · ‎08-04-2015

Hi Andrew,

There is actually a tool developed by ESRI staffers that will do this for you, produced by Jake Skinner.

http://epro.maps.arcgis.com/home/item.html?id=16e5cc64178941839eca62837f168ec9

AndrewCallison · ‎08-07-2015

Thank's Ian Murray for the link.

I am using Python34 so I had to convert the script using 2to3.py. Now I am getting this error belo

import arcpy, urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, json, os, math, sys
from arcpy import env
env.overwriteOutput = 1
env.workspace = env.scratchGDB
hostedFeatureService = arcpy.GetParameterAsText(0)
agsService = arcpy.GetParameterAsText(1)
baseURL = arcpy.GetParameterAsText(2) + "/query"
agsFeatures = arcpy.GetParameterAsText(3)
agsTable = arcpy.GetParameterAsText(4)
username = arcpy.GetParameterAsText(5)
password = arcpy.GetParameterAsText(6)
# Generate token for hosted feature service
if hostedFeatureService == 'true':
    try:
        arcpy.AddMessage('\nGenerating Token\n')
        tokenURL = 'https://www.arcgis.com/sharing/rest/generateToken'
        params = {'f': 'pjson', 'username': username, 'password': password, 'referer': 'http://www.arcgis.com'}
        req = urllib.request.Request(tokenURL, urllib.parse.urlencode(params))
        response = urllib.request.urlopen(req)
        data = json.load(response).encode("utf-8")
        token = data['token']
    except:
        token = ''
# Genereate token for AGS feature service
if agsService == 'true':
    try:
        arcpy.AddMessage('\nGenerating Token\n')
        server = baseURL.split("//")[1].split("/")[0]        
        tokenURL = 'http://' + server + '/arcgis/tokens/?username=' + username + '&password=' + password + '&referer=http%3A%2F%2F' + server + '&f=json'
        req = urllib.request.Request(tokenURL)
        response = urllib.request.urlopen(req)
        data = json.load(response).encode("utf-8")
        token = data['token']
    except:
        token = ''
        pass
    
# Return largest ObjectID
params = {'where': '1=1', 'returnIdsOnly': 'true', 'token': token, 'f': 'json'}
req = urllib.request.Request(baseURL, urllib.parse.urlencode(params))
response = urllib.request.urlopen(req)
data = json.load(response).encode("utf-8")
try:
    data['objectIds'].sort()
except:
    arcpy.AddWarning("\nURL is incorrect.  Or, Service is secure, please enter username and password.\n")
iteration = int(data['objectIds'][-1])
minOID = int(data['objectIds'][0]) - 1
OID = data['objectIdFieldName']
# Code for downloading hosted feature service
if hostedFeatureService == 'true':
    if iteration < 1000:
        x = iteration
        y = minOID
        where = OID + '>' + str(y) + 'AND ' + OID + '<=' + str(x)
        fields ='*'
       
        query = "?where={}&outFields={}&returnGeometry=true&f=json&token={}".format(where, fields, token)
        fsURL = baseURL + query
     
        fs = arcpy.FeatureSet()
        fs.load(fsURL)
        arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
        outputFC = arcpy.GetParameterAsText(7)
        desc = arcpy.Describe(os.path.dirname(outputFC))            
        if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
            outputFC2 = outputFC.split(".")[-1]
            arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), outputFC2)
        else:
            arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), os.path.basename(outputFC))       
    else:
        newIteration = (math.ceil(iteration/1000.0) * 1000) 
        x = minOID + 1000
        y = minOID
        firstTime = 'True'
        while x <= newIteration:    
            where = OID + '>' + str(y) + 'AND ' + OID + '<=' + str(x)
            fields ='*'
            query = "?where={}&outFields={}&returnGeometry=true&f=json&token={}".format(where, fields, token)
            fsURL = baseURL + query            
         
            fs = arcpy.FeatureSet()
            fs.load(fsURL)
            if firstTime == 'True':
                arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                outputFC = arcpy.GetParameterAsText(7)                
                desc = arcpy.Describe(os.path.dirname(outputFC))            
                if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
                    outputFC2 = outputFC.split(".")[-1]
                    arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), outputFC2)
                else:
                    arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), os.path.basename(outputFC))
                firstTime = 'False'
            else:
                desc = arcpy.Describe(os.path.dirname(outputFC))            
                if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
                    arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                    insertRows = arcpy.da.InsertCursor(outputFC, ["*","SHAPE@"])
                    searchRows = arcpy.da.SearchCursor(fs, ["*","SHAPE@"])
                    for searchRow in searchRows:
                        fieldList = list(searchRow)                    
                        insertRows.insertRow(fieldList) 
                elif desc.workspaceFactoryProgID == '':
                    arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                    arcpy.Append_management(fs, outputFC, "NO_TEST")
                else:
                    arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                    arcpy.Append_management(fs, outputFC)      
            x += 1000
            y += 1000
    try:
        del searchRow, searchRows, insertRows
    except:
        pass
# Check to see if downloading a feature or tabular data from a ArcGIS Server service
if agsService == 'true':
    if agsFeatures != 'true' and agsTable != 'true':
        arcpy.AddError("\nPlease check 'Downloading Feature Data' or 'Downloading Tabular Data'\n")
# Code for downloading feature data
if agsFeatures == 'true':
    if iteration < 1000:
        x = iteration
        y = minOID
        where = OID + '>' + str(y) + 'AND ' + OID + '<=' + str(x)
        fields ='*'
       
        query = "?where={}&outFields={}&returnGeometry=true&f=json&token={}".format(where, fields, token)
        fsURL = baseURL + query     
        fs = arcpy.FeatureSet()
        fs.load(fsURL)
        arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
        outputFC = arcpy.GetParameterAsText(7)
        desc = arcpy.Describe(os.path.dirname(outputFC))            
        if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
            outputFC2 = outputFC.split(".")[-1]
            arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), outputFC2)
        else:
            arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), os.path.basename(outputFC))       
    else:
        newIteration = (math.ceil(iteration/1000.0) * 1000) 
        x = minOID + 1000
        y = minOID
        firstTime = 'True'
        while x <= newIteration:    
            where = OID + '>' + str(y) + 'AND ' + OID + '<=' + str(x)
            fields ='*'
            query = "?where={}&outFields={}&returnGeometry=true&f=json&token={}".format(where, fields, token)
            fsURL = baseURL + query
         
            fs = arcpy.FeatureSet()
            fs.load(fsURL)
            if firstTime == 'True':
                arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                outputFC = arcpy.GetParameterAsText(7)                
                desc = arcpy.Describe(os.path.dirname(outputFC))            
                if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
                    outputFC2 = outputFC.split(".")[-1]
                    arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), outputFC2)
                else:
                    arcpy.FeatureClassToFeatureClass_conversion(fs, os.path.dirname(outputFC), os.path.basename(outputFC))
                firstTime = 'False'
            else:
                desc = arcpy.Describe(os.path.dirname(outputFC))            
                if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
                    arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                    insertRows = arcpy.da.InsertCursor(outputFC, ["*","SHAPE@"])
                    searchRows = arcpy.da.SearchCursor(fs, ["*","SHAPE@"])
                    for searchRow in searchRows:
                        fieldList = list(searchRow)                    
                        insertRows.insertRow(fieldList) 
                elif desc.workspaceFactoryProgID == '':
                    arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                    arcpy.Append_management(fs, outputFC, "NO_TEST")
                else:
                    arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                    arcpy.Append_management(fs, outputFC)
            x += 1000
            y += 1000
    try:
        del searchRow, searchRows, insertRows
    except:
        pass
# Code for downloading tabular data
if agsTable == 'true':
    if iteration < 1000:
        x = iteration
        y = minOID
        where = OID + '>' + str(y) + 'AND ' + OID + '<=' + str(x)
        fields ='*'
       
        query = "?where={}&outFields={}&returnGeometry=true&f=json&token={}".format(where, fields, token)
        fsURL = baseURL + query
     
        fs = arcpy.RecordSet()
        fs.load(fsURL)
        arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
        outputFC = arcpy.GetParameterAsText(7)
        desc = arcpy.Describe(os.path.dirname(outputFC))            
        if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
            outputFC2 = outputFC.split(".")[-1]
            arcpy.TableToTable_conversion(fs, os.path.dirname(outputFC), outputFC2)
        else:
            arcpy.TableToTable_conversion(fs, os.path.dirname(outputFC), os.path.basename(outputFC))       
    else:
        newIteration = (math.ceil(iteration/1000.0) * 1000) 
        x = minOID + 1000
        y = minOID
        firstTime = 'True'
        while x <= newIteration:    
            where = OID + '>' + str(y) + 'AND ' + OID + '<=' + str(x)
            fields ='*'
            query = "?where={}&outFields={}&f=json&token={}".format(where, fields, token)
            fsURL = baseURL + query
         
            fs = arcpy.RecordSet()
            fs.load(fsURL)
            if firstTime == 'True':
                arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                outputFC = arcpy.GetParameterAsText(7)                
                desc = arcpy.Describe(os.path.dirname(outputFC))            
                if desc.workspaceFactoryProgID == 'esriDataSourcesGDB.SdeWorkspaceFactory.1':
                    outputFC2 = outputFC.split(".")[-1]
                    arcpy.TableToTable_conversion(fs, os.path.dirname(outputFC), outputFC2)
                else:
                    arcpy.TableToTable_conversion(fs, os.path.dirname(outputFC), os.path.basename(outputFC))
                firstTime = 'False'
            else:
                desc = arcpy.Describe(os.path.dirname(outputFC))                            
                arcpy.AddMessage('Copying features with ObjectIDs from ' + str(y) + ' to ' + str(x))
                arcpy.Append_management(fs, outputFC)        
            x += 1000
            y += 1000
    try:
        del searchRow, searchRows, insertRows
    except:
        pass
# Code for retrieving attachments
getAttachments = arcpy.GetParameterAsText(8)
if getAttachments == 'true':
    # Create Replica to retrieve attachments
    arcpy.AddMessage("\nRetrieving Attachments\n")
    cwd = arcpy.GetParameterAsText(9)
    crUrl = baseURL[0:-7] + 'createReplica'
    crValues = {'f' : 'json',
    'layers' : '0',
    'returnAttachments' : 'true',
    'token' : token }
    crData = urllib.parse.urlencode(crValues)
    crRequest = urllib.request.Request(crUrl, crData)
    crResponse = urllib.request.urlopen(crRequest)
    crJson = json.load(crResponse)
    try:
        replicaUrl = crJson['URL']
    except KeyError:
        arcpy.AddWarning("\nService does not have 'Sync' operation enabled\n")
    urllib.request.urlretrieve(replicaUrl, cwd + os.sep + 'myLayer.json')        
    f = open(cwd + os.sep + 'myLayer.json')
    lines = f.readlines()
    f.close()
    for line in lines:
        if not 'attachments' in line:
            arcpy.AddWarning("\nService does not contain attachments\n")
            os.remove(cwd + os.sep + 'myLayer.json')
            sys.exit()
    # Get Attachment
    with open(cwd + os.sep + 'myLayer.json') as data_file:
        data = json.load(data_file)
    dict = {}
    x = 0
    while x <= iteration:
        try:
            dict[data['layers'][0]['features']['attributes'][OID]] = data['layers'][0]['features']['attributes']['GlobalID']
            x += 1
        except IndexError:
            x += 1
            pass
    fc = arcpy.GetParameterAsText(7)
    arcpy.AddField_management(fc, "GlobalID_Str", "TEXT")
    for key in dict:
        with arcpy.da.UpdateCursor(fc, [OID, "GlobalID_Str"], OID + " = " + str(key)) as cursor:
            for row in cursor:
                row[1] = dict[key]
                cursor.updateRow(row)
    arcpy.EnableAttachments_management(fc)
    arcpy.AddField_management(fc + "__ATTACH", "GlobalID_Str", "TEXT")
    arcpy.AddField_management(fc + "__ATTACH", "PhotoPath", "TEXT")                          
    # Add Attachments
    # Create Match Table
    try:
        for x in data['layers'][0]['attachments']:
            gaUrl = x['url']
            gaFolder = cwd + os.sep + x['parentGlobalId']
            if not os.path.exists(gaFolder):
                os.makedirs(gaFolder)
                gaName = x['name']
                gaValues = {'token' : token }
                gaData = urllib.parse.urlencode(gaValues)
                urllib.request.urlretrieve(url=gaUrl + '/' + gaName, filename=os.path.join(gaFolder, gaName),data=gaData)
        rows = arcpy.InsertCursor(fc + "__ATTACH")
        hasrow = False
        for cmtX in data['layers'][0]['attachments']:
            row = rows.newRow()
            hasrow = True
            row.setValue('GlobalID_Str', cmtX['parentGlobalId'])
            row.setValue('PhotoPath', cwd + os.sep +cmtX['parentGlobalId'] + os.sep + cmtX['name'])
            rows.insertRow(row)
        if hasrow == True:
            del row
            del rows
            arcpy.AddAttachments_management(fc, 'GlobalID_Str', fc + '__ATTACH', 'GlobalID_Str', 'PhotoPath')
        try:        
            arcpy.MakeTableView_management(fc + '__ATTACH', "tblView")
            arcpy.SelectLayerByAttribute_management("tblView", "NEW_SELECTION", "DATA_SIZE = 0")
            arcpy.DeleteRows_management("tblView")
            arcpy.DeleteField_management(fc + '__ATTACH', 'GlobalID_Str')
            arcpy.DeleteField_management(fc + '__ATTACH', 'PhotoPath')
        except:
            pass
    except KeyError:
        pass
    os.remove(cwd + os.sep + 'myLayer.json')

ERROR:

Traceback (most recent call last):

File "C:\Users\agcallis\Desktop\CopyFS_Pro.py", line 47, in <module>

response = urllib.request.urlopen(req)

File "c:\program files\arcgis pro\bin\Python\lib\urllib\request.py", line 153, in urlopen

return opener.open(url, data, timeout)

File "c:\program files\arcgis pro\bin\Python\lib\urllib\request.py", line 453, in open

req = meth(req)

File "c:\program files\arcgis pro\bin\Python\lib\urllib\request.py", line 1120, in do_request_

raise TypeError(msg)

TypeError: POST data should be bytes or an iterable of bytes. It cannot be of type str.

Failed to execute (DownloadService).

Can you or Jake Skinner help me out?

Thanks

IanMurray · ‎08-07-2015

Jake should probably help you out but I can take a look at it in a little bit.

For future reference, posting code in code blocks in Geonet makes it significantly easier to read.

See Posting Code blocks in the new GeoNet for help on that.

Also, you should be able to run Python 2.X and 3.X on the same computer, and I'm guessing the only reason you are using 3.4 is because you are running ArcGIS Pro? You should be able to run Pro and ArcGIS 10.3 on the same computer and use the Python 2.7 version of his tool.

AndrewCallison · ‎08-08-2015

I am trying to build python script to run with task scheduler but the server it will be running on currently only has ArcMap 10.0 with Python 2.6.5 or ArcGIS Pro. I thought it would be easier to convert the script to Python 3.x then try to make it work with Python 2.6.

Any help would be greatly appreciated.

simoxu · ‎08-27-2015

have you seen this post before, it seems only Pro supports python 3.4. ArcGIS Desktop and Server (AGOL is server) only supports 2.7.7

Python in ArcGIS 10.3