I was just thrown an ancient script (I'm pretty sure it was intended for 10.1?) and alerted it wasn't preserving editor tracking. This is not how I would write python or approach this, but any idea how to implement the same functionality y'all spoke about into it?
Kelly Gerrow
Leon Scott
import ConfigParser
import ast
import os
import sys
import time
import urllib2
import urllib
import json
import mimetypes
import gzip
from io import BytesIO
import string
import random
from xml.etree import ElementTree as ET
import arcpy
class AGOLHandler(object):
def __init__(self, username, password, serviceName, folderName, proxyDict):
self.headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent': ('updatehostedfeatureservice')
}
self.username = username
self.password = password
self.base_url = "https://www.arcgis.com/sharing/rest"
self.proxyDict = proxyDict
self.serviceName = serviceName
self.token = self.getToken(username, password)
self.itemID = self.findItem("Feature Service")
self.SDitemID = self.findItem("Service Definition")
self.folderName = folderName
self.folderID = self.findFolder()
def getToken(self, username, password, exp=60):
referer = "http://www.arcgis.com/"
query_dict = {'username': username,
'password': password,
'expiration': str(exp),
'client': 'referer',
'referer': referer,
'f': 'json'}
token_url = '{}/generateToken'.format(self.base_url)
token_response = self.url_request(token_url, query_dict, 'POST')
if "token" not in token_response:
print(token_response['error'])
sys.exit()
else:
return token_response['token']
def findItem(self, findType):
""" Find the itemID of whats being updated
"""
searchURL = self.base_url + "/search"
query_dict = {'f': 'json',
'token': self.token,
'q': "title:\"" + self.serviceName + "\"AND owner:\"" +
self.username + "\" AND type:\"" + findType + "\""}
jsonResponse = self.url_request(searchURL, query_dict, 'POST')
if jsonResponse['total'] == 0:
print("\nCould not find a service to update. Check the service name in the settings.ini")
sys.exit()
else:
resultList = jsonResponse['results']
for it in resultList:
if it["title"] == self.serviceName:
print("found {} : {}").format(findType, it["id"])
return it["id"]
def findFolder(self, folderName=None):
""" Find the ID of the folder containing the service
"""
if self.folderName == "None":
return ""
findURL = "{}/content/users/{}".format(self.base_url, self.username)
query_dict = {'f': 'json',
'num': 1,
'token': self.token}
jsonResponse = self.url_request(findURL, query_dict, 'POST')
for folder in jsonResponse['folders']:
if folder['title'] == self.folderName:
return folder['id']
print("\nCould not find the specified folder name provided in the settings.ini")
print("-- If your content is in the root folder, change the folder name to 'None'")
sys.exit()
def upload(self, fileName, tags, description):
"""
Overwrite the SD on AGOL with the new SD.
This method uses 3rd party module: requests
"""
updateURL = '{}/content/users/{}/{}/items/{}/update'.format(self.base_url, self.username,
self.folderID, self.SDitemID)
query_dict = {"filename": fileName,
"type": "Service Definition",
"title": self.serviceName,
"tags": tags,
"description": description,
"f": "json",
'multipart': 'true',
"token": self.token}
details = {'filename': fileName}
add_item_res = self.url_request(updateURL, query_dict, "POST", "", details)
itemPartJSON = self._add_part(fileName, add_item_res['id'], "Service Definition")
if "success" in itemPartJSON:
itemPartID = itemPartJSON['id']
commit_response = self.commit(itemPartID)
status = 'processing'
while status == 'processing' or status == 'partial':
status = self.item_status(itemPartID)['status']
time.sleep(1.5)
print("updated SD: {}".format(itemPartID))
return True
else:
print("\n.sd file not uploaded. Check the errors and try again.\n")
print(itemPartJSON)
sys.exit()
def _add_part(self, file_to_upload, item_id, upload_type=None):
""" Add the item to the portal in chunks.
"""
def read_in_chunks(file_object, chunk_size=10000000):
"""Generate file chunks of 10MB"""
while True:
data = file_object.read(chunk_size)
if not data:
break
yield data
url = '{}/content/users/{}/items/{}/addPart'.format(self.base_url, self.username, item_id)
with open(file_to_upload, 'rb') as f:
for part_num, piece in enumerate(read_in_chunks(f), start=1):
title = os.path.basename(file_to_upload)
files = {"file": {"filename": file_to_upload, "content": piece}}
params = {
'f': "json",
'token': self.token,
'partNum': part_num,
'title': title,
'itemType': 'file',
'type': upload_type
}
request_data, request_headers = self.multipart_request(params, files)
resp = self.url_request(url, request_data, "MULTIPART", request_headers)
return resp
def item_status(self, item_id):
""" Gets the status of an item.
Returns:
The item's status. (partial | processing | failed | completed)
"""
url = '{}/content/users/{}/items/{}/status'.format(self.base_url, self.username, item_id)
parameters = {'token': self.token,
'f': 'json'}
return self.url_request(url, parameters)
def commit(self, item_id):
""" Commits an item that was uploaded as multipart
"""
url = '{}/content/users/{}/items/{}/commit'.format(self.base_url, self.username, item_id)
parameters = {'token': self.token,
'f': 'json'}
return self.url_request(url, parameters)
def publish(self):
""" Publish the existing SD on AGOL (it will be turned into a Feature Service)
"""
publishURL = '{}/content/users/{}/publish'.format(self.base_url, self.username)
query_dict = {'itemID': self.SDitemID,
'filetype': 'serviceDefinition',
'overwrite': 'true',
'f': 'json',
'token': self.token}
jsonResponse = self.url_request(publishURL, query_dict, 'POST')
try:
if not jsonResponse['services'][0]['success']:
print("Cannot proceed with publishing, error: \n {}".format(
jsonResponse['services'][0]['error']))
sys.exit()
else:
print("successfully updated...{}...".format(jsonResponse['services']))
except Exception as e:
print("Problem reading response: \n {}".format(e))
return jsonResponse['services'][0]['serviceItemId']
def enableSharing(self, newItemID, everyone, orgs, groups):
""" Share an item with everyone, the organization and/or groups
"""
shareURL = '{}/content/users/{}/{}/items/{}/share'.format(self.base_url, self.username,
self.folderID, newItemID)
if groups is None:
groups = ''
query_dict = {'f': 'json',
'everyone': everyone,
'org': orgs,
'groups': groups,
'token': self.token}
jsonResponse = self.url_request(shareURL, query_dict, 'POST')
print("successfully shared...{}...".format(jsonResponse['itemId']))
def url_request(self, in_url, request_parameters, request_type='GET',
additional_headers=None, files=None, repeat=0):
"""
Make a request to the portal, provided a portal URL
and request parameters, returns portal response.
Arguments:
in_url -- portal url
request_parameters -- dictionary of request parameters.
request_type -- HTTP verb (default: GET)
additional_headers -- any headers to pass along with the request.
files -- any files to send.
repeat -- repeat the request up to this number of times.
Returns:
dictionary of response from portal instance.
"""
if request_type == 'GET':
req = urllib2.Request('?'.join((in_url, urllib.urlencode(request_parameters))))
elif request_type == 'MULTIPART':
req = urllib2.Request(in_url, request_parameters)
else:
req = urllib2.Request(
in_url, urllib.urlencode(request_parameters), self.headers)
if additional_headers:
for key, value in list(additional_headers.items()):
req.add_header(key, value)
req.add_header('Accept-encoding', 'gzip')
if self.proxyDict:
p = urllib2.ProxyHandler(self.proxyDict)
auth = urllib2.HTTPBasicAuthHandler()
opener = urllib2.build_opener(p, auth, urllib2.HTTPHandler)
urllib2.install_opener(opener)
response = urllib2.urlopen(req)
if response.info().get('Content-Encoding') == 'gzip':
buf = BytesIO(response.read())
with gzip.GzipFile(fileobj=buf) as gzip_file:
response_bytes = gzip_file.read()
else:
response_bytes = response.read()
response_text = response_bytes.decode('UTF-8')
response_json = json.loads(response_text)
if not response_json or "error" in response_json:
rerun = False
if repeat > 0:
repeat -= 1
rerun = True
if rerun:
time.sleep(2)
response_json = self.url_request(
in_url, request_parameters, request_type,
additional_headers, files, repeat)
return response_json
def multipart_request(self, params, files):
""" Uploads files as multipart/form-data. files is a dict and must
contain the required keys "filename" and "content". The "mimetype"
value is optional and if not specified will use mimetypes.guess_type
to determine the type or use type application/octet-stream. params
is a dict containing the parameters to be passed in the HTTP
POST request.
content = open(file_path, "rb").read()
files = {"file": {"filename": "some_file.sd", "content": content}}
params = {"f": "json", "token": token, "type": item_type,
"title": title, "tags": tags, "description": description}
data, headers = multipart_request(params, files)
"""
letters_digits = "".join(string.digits + string.ascii_letters)
boundary = "----WebKitFormBoundary{}".format("".join(random.choice(letters_digits) for i in range(16)))
file_lines = []
for name, value in params.iteritems():
file_lines.extend(("--{}".format(boundary),
'Content-Disposition: form-data; name="{}"'.format(name),
"", str(value)))
for name, value in files.items():
if "filename" in value:
filename = value.get("filename")
else:
raise Exception("The filename key is required.")
if "mimetype" in value:
mimetype = value.get("mimetype")
else:
mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
if "content" in value:
file_lines.extend(("--{}".format(boundary),
'Content-Disposition: form-data; name="{}"; filename="{}"'.format(name, filename),
"Content-Type: {}".format(mimetype), "",
(value.get("content"))))
else:
raise Exception("The content key is required.")
file_lines.extend(("--{}--".format(boundary), ""))
request_data = "\r\n".join(file_lines)
request_headers = {"Content-Type": "multipart/form-data; boundary={}".format(boundary),
"Content-Length": str(len(request_data))}
return request_data, request_headers
def makeSD(MXD, serviceName, tempDir, outputSD, maxRecords, tags, summary):
""" create a draft SD and modify the properties to overwrite an existing FS
"""
arcpy.env.overwriteOutput = True
SDdraft = os.path.join(tempDir, "tempdraft.sddraft")
newSDdraft = os.path.join(tempDir, "updatedDraft.sddraft")
try:
mappingMXD = arcpy.mapping.MapDocument(MXD)
if mappingMXD.tags == "":
mappingMXD.tags = tags
mappingMXD.save()
if mappingMXD.summary == "":
mappingMXD.summary = summary
mappingMXD.save()
except IOError:
print("IOError on save, do you have the MXD open? Summary/tag info not pushed to MXD, publishing may fail.")
arcpy.mapping.CreateMapSDDraft(MXD, SDdraft, serviceName, "MY_HOSTED_SERVICES")
doc = ET.parse(SDdraft)
root_elem = doc.getroot()
if root_elem.tag != "SVCManifest":
raise ValueError("Root tag is incorrect. Is {} a .sddraft file?".format(SDDraft))
for config in doc.findall("./Configurations/SVCConfiguration/TypeName"):
if config.text == "MapServer":
config.text = "FeatureServer"
for prop in doc.findall("./Configurations/SVCConfiguration/Definition/" +
"ConfigurationProperties/PropertyArray/" +
"PropertySetProperty"):
if prop.find("Key").text == 'isCached':
prop.find("Value").text = "false"
if prop.find("Key").text == 'maxRecordCount':
prop.find("Value").text = maxRecords
for prop in doc.findall("./Configurations/SVCConfiguration/Definition/Info/PropertyArray/PropertySetProperty"):
if prop.find("Key").text == 'WebCapabilities':
prop.find("Value").text = "Query"
root_elem.attrib["xmlns:typens"] = 'http://www.esri.com/schemas/ArcGIS/10.1'
root_elem.attrib["xmlns:xs"] = 'http://www.w3.org/2001/XMLSchema'
with open(newSDdraft, 'w') as f:
doc.write(f, 'utf-8')
analysis = arcpy.mapping.AnalyzeForSD(newSDdraft)
if analysis['errors'] == {}:
arcpy.StageService_server(newSDdraft, outputSD)
print("Created {}".format(outputSD))
else:
print("Errors in analyze: \n {}".format(analysis['errors']))
sys.exit()
if __name__ == "__main__":
print("Starting Feature Service publish process")
localPath = sys.path[0]
settingsFile = os.path.join(localPath, "settings.ini")
if os.path.isfile(settingsFile):
config = ConfigParser.ConfigParser()
config.read(settingsFile)
else:
print("INI file not found. \nMake sure a valid 'settings.ini' file exists in the same directory as this script.")
sys.exit()
inputUsername = config.get('AGOL', 'USER')
inputPswd = config.get('AGOL', 'PASS')
MXD = config.get('FS_INFO', 'MXD')
serviceName = config.get('FS_INFO', 'SERVICENAME')
folderName = config.get('FS_INFO', 'FOLDERNAME')
tags = config.get('FS_INFO', 'TAGS')
summary = config.get('FS_INFO', 'DESCRIPTION')
maxRecords = config.get('FS_INFO', 'MAXRECORDS')
shared = config.get('FS_SHARE', 'SHARE')
everyone = config.get('FS_SHARE', 'EVERYONE')
orgs = config.get('FS_SHARE', 'ORG')
groups = config.get('FS_SHARE', 'GROUPS')
use_prxy = config.get('PROXY', 'USEPROXY')
pxy_srvr = config.get('PROXY', 'SERVER')
pxy_port = config.get('PROXY', 'PORT')
pxy_user = config.get('PROXY', 'USER')
pxy_pass = config.get('PROXY', 'PASS')
proxyDict = {}
if ast.literal_eval(use_prxy):
http_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port
https_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port
ftp_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port
proxyDict = {"http": http_proxy, "https": https_proxy, "ftp": ftp_proxy}
tempDir = os.path.join(localPath, "tempDir")
if not os.path.isdir(tempDir):
os.mkdir(tempDir)
finalSD = os.path.join(tempDir, serviceName + ".sd")
agol = AGOLHandler(inputUsername, inputPswd, serviceName, folderName, proxyDict)
makeSD(MXD, serviceName, tempDir, finalSD, maxRecords, tags, summary)
if agol.upload(finalSD, tags, summary):
fsID = agol.publish()
if ast.literal_eval(shared):
agol.enableSharing(fsID, everyone, orgs, groups)
print("\nfinished.")
I have no idea why this relatively simple task was written in 500 lines of python.