1#!/usr/bin/python
2#
3# Copyright 2007 Google Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18An interactive, stateful AJAX shell that runs Python code on the server.
19
20Part of http://code.google.com/p/google-app-engine-samples/.
21
22May be run as a standalone app or in an existing app as an admin-only handler.
23Can be used for system administration tasks, as an interactive way to try out
24APIs, or as a debugging aid during development.
25
26The logging, os, sys, db, and users modules are imported automatically.
27
28Interpreter state is stored in the datastore so that variables, function
29definitions, and other values in the global and local namespaces can be used
30across commands.
31
32To use the shell in your app, copy shell.py, static/*, and templates/* into
33your app's source directory. Then, copy the URL handlers from app.yaml into
34your app.yaml.
35
36TODO: unit tests!
37"""
38
39import logging
40import new
41import os
42import pickle
43import sys
44import traceback
45import types
46import wsgiref.handlers
47
48from google.appengine.api import users
49from google.appengine.ext import db
50from google.appengine.ext import webapp
51from google.appengine.ext.webapp import template
52
53
54# Set to True if stack traces should be shown in the browser, etc.
55_DEBUG = True
56
57# The entity kind for shell sessions. Feel free to rename to suit your app.
58_SESSION_KIND = '_Shell_Session'
59
60# Types that can't be pickled.
61UNPICKLABLE_TYPES = (
62  types.ModuleType,
63  types.TypeType,
64  types.ClassType,
65  types.FunctionType,
66  )
67
68# Unpicklable statements to seed new sessions with.
69INITIAL_UNPICKLABLES = [
70  'import logging',
71  'import os',
72  'import sys',
73  'from google.appengine.ext import db',
74  'from google.appengine.api import users',
75  ]
76
77
78class Session(db.Model):
79  """A shell session. Stores the session's globals.
80
81  Each session globals is stored in one of two places:
82
83  If the global is picklable, it's stored in the parallel globals and
84  global_names list properties. (They're parallel lists to work around the
85  unfortunate fact that the datastore can't store dictionaries natively.)
86
87  If the global is not picklable (e.g. modules, classes, and functions), or if
88  it was created by the same statement that created an unpicklable global,
89  it's not stored directly. Instead, the statement is stored in the
90  unpicklables list property. On each request, before executing the current
91  statement, the unpicklable statements are evaluated to recreate the
92  unpicklable globals.
93
94  The unpicklable_names property stores all of the names of globals that were
95  added by unpicklable statements. When we pickle and store the globals after
96  executing a statement, we skip the ones in unpicklable_names.
97
98  Using Text instead of string is an optimization. We don't query on any of
99  these properties, so they don't need to be indexed.
100  """
101  global_names = db.ListProperty(db.Text)
102  globals = db.ListProperty(db.Blob)
103  unpicklable_names = db.ListProperty(db.Text)
104  unpicklables = db.ListProperty(db.Text)
105
106  def set_global(self, name, value):
107    """Adds a global, or updates it if it already exists.
108
109    Also removes the global from the list of unpicklable names.
110
111    Args:
112      name: the name of the global to remove
113      value: any picklable value
114    """
115    blob = db.Blob(pickle.dumps(value))
116
117    if name in self.global_names:
118      index = self.global_names.index(name)
119      self.globals[index] = blob
120    else:
121      self.global_names.append(db.Text(name))
122      self.globals.append(blob)
123
124    self.remove_unpicklable_name(name)
125
126  def remove_global(self, name):
127    """Removes a global, if it exists.
128
129    Args:
130      name: string, the name of the global to remove
131    """
132    if name in self.global_names:
133      index = self.global_names.index(name)
134      del self.global_names[index]
135      del self.globals[index]
136
137  def globals_dict(self):
138    """Returns a dictionary view of the globals.
139    """
140    return dict((name, pickle.loads(val))
141                for name, val in zip(self.global_names, self.globals))
142
143  def add_unpicklable(self, statement, names):
144    """Adds a statement and list of names to the unpicklables.
145
146    Also removes the names from the globals.
147
148    Args:
149      statement: string, the statement that created new unpicklable global(s).
150      names: list of strings; the names of the globals created by the statement.
151    """
152    self.unpicklables.append(db.Text(statement))
153
154    for name in names:
155      self.remove_global(name)
156      if name not in self.unpicklable_names:
157        self.unpicklable_names.append(db.Text(name))
158
159  def remove_unpicklable_name(self, name):
160    """Removes a name from the list of unpicklable names, if it exists.
161
162    Args:
163      name: string, the name of the unpicklable global to remove
164    """
165    if name in self.unpicklable_names:
166      self.unpicklable_names.remove(name)
167
168
169class FrontPageHandler(webapp.RequestHandler):
170  """Creates a new session and renders the shell.html template.
171  """
172
173  def get(self):
174    # set up the session. TODO: garbage collect old shell sessions
175    session_key = self.request.get('session')
176    if session_key:
177      session = Session.get(session_key)
178    else:
179      # create a new session
180      session = Session()
181      session.unpicklables = [db.Text(line) for line in INITIAL_UNPICKLABLES]
182      session_key = session.put()
183
184    template_file = os.path.join(os.path.dirname(__file__), 'templates',
185                                 'shell.html')
186    session_url = '/?session=%s' % session_key
187    vars = { 'server_software': os.environ['SERVER_SOFTWARE'],
188             'python_version': sys.version,
189             'session': str(session_key),
190             'user': users.get_current_user(),
191             'login_url': users.create_login_url(session_url),
192             'logout_url': users.create_logout_url(session_url),
193             }
194    rendered = webapp.template.render(template_file, vars, debug=_DEBUG)
195    self.response.out.write(rendered)
196
197
198class StatementHandler(webapp.RequestHandler):
199  """Evaluates a python statement in a given session and returns the result.
200  """
201
202  def get(self):
203    self.response.headers['Content-Type'] = 'text/plain'
204
205    # extract the statement to be run
206    statement = self.request.get('statement')
207    if not statement:
208      return
209
210    # the python compiler doesn't like network line endings
211    statement = statement.replace('\r\n', '\n')
212
213    # add a couple newlines at the end of the statement. this makes
214    # single-line expressions such as 'class Foo: pass' evaluate happily.
215    statement += '\n\n'
216
217    # log and compile the statement up front
218    try:
219      logging.info('Compiling and evaluating:\n%s' % statement)
220      compiled = compile(statement, '<string>', 'single')
221    except:
222      self.response.out.write(traceback.format_exc())
223      return
224
225    # create a dedicated module to be used as this statement's __main__
226    statement_module = new.module('__main__')
227
228    # use this request's __builtin__, since it changes on each request.
229    # this is needed for import statements, among other things.
230    import __builtin__
231    statement_module.__builtins__ = __builtin__
232
233    # load the session from the datastore
234    session = Session.get(self.request.get('session'))
235
236    # swap in our custom module for __main__. then unpickle the session
237    # globals, run the statement, and re-pickle the session globals, all
238    # inside it.
239    old_main = sys.modules.get('__main__')
240    try:
241      sys.modules['__main__'] = statement_module
242      statement_module.__name__ = '__main__'
243
244      # re-evaluate the unpicklables
245      for code in session.unpicklables:
246        exec code in statement_module.__dict__
247
248      # re-initialize the globals
249      for name, val in session.globals_dict().items():
250        try:
251          statement_module.__dict__[name] = val
252        except:
253          msg = 'Dropping %s since it could not be unpickled.\n' % name
254          self.response.out.write(msg)
255          logging.warning(msg + traceback.format_exc())
256          session.remove_global(name)
257
258      # run!
259      old_globals = dict(statement_module.__dict__)
260      try:
261        old_stdout = sys.stdout
262        old_stderr = sys.stderr
263        try:
264          sys.stdout = self.response.out
265          sys.stderr = self.response.out
266          exec compiled in statement_module.__dict__
267        finally:
268          sys.stdout = old_stdout
269          sys.stderr = old_stderr
270      except:
271        self.response.out.write(traceback.format_exc())
272        return
273
274      # extract the new globals that this statement added
275      new_globals = {}
276      for name, val in statement_module.__dict__.items():
277        if name not in old_globals or val != old_globals[name]:
278          new_globals[name] = val
279
280      if True in [isinstance(val, UNPICKLABLE_TYPES)
281                  for val in new_globals.values()]:
282        # this statement added an unpicklable global. store the statement and
283        # the names of all of the globals it added in the unpicklables.
284        session.add_unpicklable(statement, new_globals.keys())
285        logging.debug('Storing this statement as an unpicklable.')
286
287      else:
288        # this statement didn't add any unpicklables. pickle and store the
289        # new globals back into the datastore.
290        for name, val in new_globals.items():
291          if not name.startswith('__'):
292            session.set_global(name, val)
293
294    finally:
295      sys.modules['__main__'] = old_main
296
297    session.put()
298
299
300def main():
301  application = webapp.WSGIApplication(
302    [('/gae_shell/', FrontPageHandler),
303     ('/gae_shell/shell.do', StatementHandler)], debug=_DEBUG)
304  wsgiref.handlers.CGIHandler().run(application)
305
306
307if __name__ == '__main__':
308  main()
309