Fix generation for methods with abnormal page token conventions (googleapis#330)

* Fix generation for methods with abnormal page token conventions

Addresses googleapis/gapic-generator#692

Author: tcoffee-google

googleapiclient/discovery.py

@@ -117,6 +117,7 @@
     'type': 'string',
     'required': False,
 }
+_PAGE_TOKEN_NAMES = ('pageToken', 'nextPageToken')
 
 # Parameters accepted by the stack, but not visible via discovery.
 # TODO(dhermes): Remove 'userip' in 'v2'.
@@ -724,7 +725,11 @@ def method(self, **kwargs):
 
     for name in parameters.required_params:
       if name not in kwargs:
-        raise TypeError('Missing required parameter "%s"' % name)
+        # temporary workaround for non-paging methods incorrectly requiring
+        # page token parameter (cf. drive.changes.watch vs. drive.changes.list)
+        if name not in _PAGE_TOKEN_NAMES or _findPageTokenName(
+            _methodProperties(methodDesc, schema, 'response')):
+          raise TypeError('Missing required parameter "%s"' % name)
 
     for name, regex in six.iteritems(parameters.pattern_params):
       if name in kwargs:
@@ -927,13 +932,20 @@ def method(self, **kwargs):
   return (methodName, method)
 
 
-def createNextMethod(methodName):
+def createNextMethod(methodName,
+                     pageTokenName='pageToken',
+                     nextPageTokenName='nextPageToken',
+                     isPageTokenParameter=True):
   """Creates any _next methods for attaching to a Resource.
 
   The _next methods allow for easy iteration through list() responses.
 
   Args:
     methodName: string, name of the method to use.
+    pageTokenName: string, name of request page token field.
+    nextPageTokenName: string, name of response page token field.
+    isPageTokenParameter: Boolean, True if request page token is a query
+        parameter, False if request page token is a field of the request body.
   """
   methodName = fix_method_name(methodName)
 
@@ -951,24 +963,24 @@ def methodNext(self, previous_request, previous_response):
     # Retrieve nextPageToken from previous_response
     # Use as pageToken in previous_request to create new request.
 
-    if 'nextPageToken' not in previous_response or not previous_response['nextPageToken']:
+    nextPageToken = previous_response.get(nextPageTokenName, None)
+    if not nextPageToken:
       return None
 
     request = copy.copy(previous_request)
 
-    pageToken = previous_response['nextPageToken']
-    parsed = list(urlparse(request.uri))
-    q = parse_qsl(parsed[4])
-
-    # Find and remove old 'pageToken' value from URI
-    newq = [(key, value) for (key, value) in q if key != 'pageToken']
-    newq.append(('pageToken', pageToken))
-    parsed[4] = urlencode(newq)
-    uri = urlunparse(parsed)
-
-    request.uri = uri
-
-    logger.info('URL being requested: %s %s' % (methodName,uri))
+    if isPageTokenParameter:
+        # Replace pageToken value in URI
+        request.uri = _add_query_parameter(
+            request.uri, pageTokenName, nextPageToken)
+        logger.info('Next page request URL: %s %s' % (methodName, request.uri))
+    else:
+        # Replace pageToken value in request body
+        model = self._model
+        body = model.deserialize(request.body)
+        body[pageTokenName] = nextPageToken
+        request.body = model.serialize(body)
+        logger.info('Next page request body: %s %s' % (methodName, body))
 
     return request
 
@@ -1116,19 +1128,59 @@ def methodResource(self):
                                method.__get__(self, self.__class__))
 
   def _add_next_methods(self, resourceDesc, schema):
-    # Add _next() methods
-    # Look for response bodies in schema that contain nextPageToken, and methods
-    # that take a pageToken parameter.
-    if 'methods' in resourceDesc:
-      for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
-        if 'response' in methodDesc:
-          responseSchema = methodDesc['response']
-          if '$ref' in responseSchema:
-            responseSchema = schema.get(responseSchema['$ref'])
-          hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
-                                                                   {})
-          hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
-          if hasNextPageToken and hasPageToken:
-            fixedMethodName, method = createNextMethod(methodName + '_next')
-            self._set_dynamic_attr(fixedMethodName,
-                                   method.__get__(self, self.__class__))
+    # Add _next() methods if and only if one of the names 'pageToken' or
+    # 'nextPageToken' occurs among the fields of both the method's response
+    # type either the method's request (query parameters) or request body.
+    if 'methods' not in resourceDesc:
+      return
+    for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
+      nextPageTokenName = _findPageTokenName(
+          _methodProperties(methodDesc, schema, 'response'))
+      if not nextPageTokenName:
+        continue
+      isPageTokenParameter = True
+      pageTokenName = _findPageTokenName(methodDesc.get('parameters', {}))
+      if not pageTokenName:
+        isPageTokenParameter = False
+        pageTokenName = _findPageTokenName(
+            _methodProperties(methodDesc, schema, 'request'))
+      if not pageTokenName:
+        continue
+      fixedMethodName, method = createNextMethod(
+          methodName + '_next', pageTokenName, nextPageTokenName,
+          isPageTokenParameter)
+      self._set_dynamic_attr(fixedMethodName,
+                             method.__get__(self, self.__class__))
+
+
+def _findPageTokenName(fields):
+  """Search field names for one like a page token.
+
+  Args:
+    fields: container of string, names of fields.
+
+  Returns:
+    First name that is either 'pageToken' or 'nextPageToken' if one exists,
+    otherwise None.
+  """
+  return next((tokenName for tokenName in _PAGE_TOKEN_NAMES
+              if tokenName in fields), None)
+
+def _methodProperties(methodDesc, schema, name):
+  """Get properties of a field in a method description.
+
+  Args:
+    methodDesc: object, fragment of deserialized discovery document that
+      describes the method.
+    schema: object, mapping of schema names to schema descriptions.
+    name: string, name of top-level field in method description.
+
+  Returns:
+    Object representing fragment of deserialized discovery document
+    corresponding to 'properties' field of object corresponding to named field
+    in method description, if it exists, otherwise empty dict.
+  """
+  desc = methodDesc.get(name, {})
+  if '$ref' in desc:
+    desc = schema.get(desc['$ref'], {})
+  return desc.get('properties', {})

googleapiclient/http.py

@@ -817,6 +817,7 @@ def execute(self, http=None, num_retries=0):
     if 'content-length' not in self.headers:
       self.headers['content-length'] = str(self.body_size)
     # If the request URI is too long then turn it into a POST request.
+    # Assume that a GET request never contains a request body.
     if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
       self.method = 'POST'
       self.headers['x-http-method-override'] = 'GET'

googleapiclient/schema.py

@@ -161,13 +161,14 @@ def prettyPrintSchema(self, schema):
     # Return with trailing comma and newline removed.
     return self._prettyPrintSchema(schema, dent=1)[:-2]
 
-  def get(self, name):
+  def get(self, name, default=None):
     """Get deserialized JSON schema from the schema name.
 
     Args:
       name: string, Schema name.
+      default: object, return value if name not found.
     """
-    return self.schemas[name]
+    return self.schemas.get(name, default)
 
 
 class _SchemaToStruct(object):