ob-python: Results handling for dicts, dataframes, arrays, plots

Thanks to Ihor Radchenko and Liu Hui for valuable feedback on this
patch.

* lisp/ob-python.el (org-babel-execute:python): Parse graphics-file
from params, and pass it to `org-babel-python-evaluate'.
(org-babel-python--output-graphics-wrapper): New constant.  Python
code to save graphical output.
(org-babel-python--def-format-value): New constant.  Python function
to format and save value results to file.  Includes handling for
graphics, dataframes, and arrays.
(org-babel-python-table-or-string): Prevent `org-babel-script-escape'
from mangling dict results.
(org-babel-python--setup-session): New function to
run setup code within Python session.
(org-babel-python-initiate-session-by-key): Replace lambda with
`org-babel-python--setup-session' in `python-shell-first-prompt-hook',
or run it directly if the session was started outside Org.
(org-babel-python-wrapper-method): Removed.  Instead use part of the
string directly in `org-babel-python-evaluate-external-process'.
(org-babel-python-pp-wrapper-method): Removed.  Pretty printing is now
handled by `org-babel-python--def-format-value'.
(org-babel-python--exec-tmpfile): Removed.  Instead use the raw string
directly in `org-babel-python-evaluate-session'.
(org-babel-python-format-session-value): Updated to use
`org-babel-python--def-format-value' for formatting value result.
(org-babel-python-evaluate): New parameter graphics-file.  Pass
graphics-file onto downstream helper functions.
(org-babel-python-evaluate-external-process): New parameter
graphics-file.  Use `org-babel-python--output-graphics-wrapper' for
graphical output.  For value result, use
`org-babel-python--def-format-value'.  Don't call `org-trim' on
results, to prevent misalignment of pandas DataFrames.
(org-babel-python-evaluate-session): New parameter graphics-file.  Use
`org-babel-python--output-graphics-wrapper' for graphical output.
Replace the removed constant `org-babel-python--exec-tmpfile' with the
string directly.  Rename local variable tmp-results-file to
results-file, which may take the value of graphics-file when provided.
(org-babel-python-async-evaluate-session): New parameter
graphics-file.  Use `org-babel-python--output-graphics-wrapper' for
graphical output.  Rename local variable tmp-results-file to
results-file, which may take the value of graphics-file when provided.

etc/ORG-NEWS

@@ -576,6 +576,23 @@ of all relational operators (~<*~, ~=*~, ~!=*~, etc.) that work like
 the regular, unstarred operators but match a headline only if the
 tested property is actually present.
 
+*** =ob-python.el=: Support for more result types and plotting
+
+=ob-python= now converts the following objects to org-mode tables when
+":results table" header arg is set:
+
+- Dictionaries
+- Numpy arrays
+- Pandas DataFrames
+- Pandas Series
+
+When the header argument =:results graphics= is set, =ob-python= will
+use matplotlib to save graphics. The behavior depends on whether value
+or output results are used. For value results, the last line should
+return a matplotlib Figure object to plot. For output results, the
+current figure (as returned by =pyplot.gcf()=) is cleared before
+evaluation, and then plotted afterwards.
+
 ** New functions and changes in function arguments
 *** =TYPES= argument in ~org-element-lineage~ can now be a symbol
 

lisp/ob-python.el

@@ -70,6 +70,8 @@ This function is called by `org-babel-execute-src-block'."
 	      org-babel-python-command))
 	 (session (org-babel-python-initiate-session
 		   (cdr (assq :session params))))
+	 (graphics-file (and (member "graphics" (assq :result-params params))
+			     (org-babel-graphical-output-file params)))
          (result-params (cdr (assq :result-params params)))
          (result-type (cdr (assq :result-type params)))
 	 (return-val (when (eq result-type 'value)
@@ -85,7 +87,7 @@ This function is called by `org-babel-execute-src-block'."
 	     (format (if session "\n%s" "\nreturn %s") return-val))))
          (result (org-babel-python-evaluate
 		  session full-body result-type
-		  result-params preamble async)))
+		  result-params preamble async graphics-file)))
     (org-babel-reassemble-table
      result
      (org-babel-pick-name (cdr (assq :colname-names params))
@@ -117,6 +119,60 @@ VARS contains resolved variable references."
 
 ;; helper functions
 
+(defconst org-babel-python--output-graphics-wrapper "\
+import matplotlib.pyplot
+matplotlib.pyplot.gcf().clear()
+%s
+matplotlib.pyplot.savefig('%s')"
+  "Format string for saving Python graphical output.
+Has two %s escapes, for the Python code to be evaluated, and the
+file to save the graphics to.")
+
+(defconst org-babel-python--def-format-value "\
+def __org_babel_python_format_value(result, result_file, result_params):
+    with open(result_file, 'w') as f:
+        if 'graphics' in result_params:
+            result.savefig(result_file)
+        elif 'pp' in result_params:
+            import pprint
+            f.write(pprint.pformat(result))
+        elif 'list' in result_params and isinstance(result, dict):
+            f.write(str(['{} :: {}'.format(k, v) for k, v in result.items()]))
+        else:
+            if not set(result_params).intersection(\
+['scalar', 'verbatim', 'raw']):
+                def dict2table(res):
+                    if isinstance(res, dict):
+                        return [(k, dict2table(v)) for k, v in res.items()]
+                    elif isinstance(res, list) or isinstance(res, tuple):
+                        return [dict2table(x) for x in res]
+                    else:
+                        return res
+                if 'table' in result_params:
+                    result = dict2table(result)
+                try:
+                    import pandas
+                except ImportError:
+                    pass
+                else:
+                    if isinstance(result, pandas.DataFrame) and 'table' in result_params:
+                        result = [[result.index.name or ''] + list(result.columns)] + \
+[None] + [[i] + list(row) for i, row in result.iterrows()]
+                    elif isinstance(result, pandas.Series) and 'table' in result_params:
+                        result = list(result.items())
+                try:
+                    import numpy
+                except ImportError:
+                    pass
+                else:
+                    if isinstance(result, numpy.ndarray):
+                        if 'table' in result_params:
+                            result = result.tolist()
+                        else:
+                            result = repr(result)
+            f.write(str(result))"
+  "Python function to format value result and save it to file.")
+
 (defun org-babel-variable-assignments:python (params)
   "Return a list of Python statements assigning the block's variables."
   (mapcar
@@ -140,9 +196,12 @@ specifying a variable of the same value."
 
 (defun org-babel-python-table-or-string (results)
   "Convert RESULTS into an appropriate elisp value.
-If the results look like a list or tuple, then convert them into an
+If the results look like a list or tuple (but not a dict), then
-Emacs-lisp table, otherwise return the results as a string."
+convert them into an Emacs-lisp table.  Otherwise return the
-  (let ((res (org-babel-script-escape results)))
+results as a string."
+  (let ((res (if (string-equal "{" (substring results 0 1))
+                 results ;don't covert dicts to elisp
+               (org-babel-script-escape results))))
     (if (listp res)
         (mapcar (lambda (el) (if (eq el 'None)
                                  org-babel-python-None-to el))
@@ -171,6 +230,13 @@ Emacs-lisp table, otherwise return the results as a string."
 
 (defvar-local org-babel-python--initialized nil
   "Flag used to mark that python session has been initialized.")
+(defun org-babel-python--setup-session ()
+  "Babel Python session setup code, to be run once per session.
+Function should be run from within the Python session buffer.
+This is often run as a part of `python-shell-first-prompt-hook',
+unless the Python session was created outside Org."
+  (python-shell-send-string-no-output org-babel-python--def-format-value)
+  (setq-local org-babel-python--initialized t))
 (defun org-babel-python-initiate-session-by-key (&optional session)
   "Initiate a python session.
 If there is not a current inferior-process-buffer in SESSION
@@ -187,23 +253,26 @@ then create.  Return the initialized session."
            (existing-session-p (comint-check-proc py-buffer)))
       (run-python cmd)
       (with-current-buffer py-buffer
-        ;; Adding to `python-shell-first-prompt-hook' immediately
+        (if existing-session-p
-        ;; after `run-python' should be safe from race conditions,
+            ;; Session was created outside Org.  Assume first prompt
-        ;; because subprocess output only arrives when Emacs is
+            ;; already happened; run session setup code directly
-        ;; waiting (see elisp manual, "Output from Processes")
+            (unless org-babel-python--initialized
-        (add-hook
+              (org-babel-python--setup-session))
-         'python-shell-first-prompt-hook
+          ;; Adding to `python-shell-first-prompt-hook' immediately
-         (lambda () (setq-local org-babel-python--initialized t))
+          ;; after `run-python' should be safe from race conditions,
-         nil 'local))
+          ;; because subprocess output only arrives when Emacs is
-      ;; Don't hang if session was started externally
+          ;; waiting (see elisp manual, "Output from Processes")
-      (unless existing-session-p
+          (add-hook
-        ;; Wait until Python initializes
+           'python-shell-first-prompt-hook
-        ;; This is more reliable compared to
+           #'org-babel-python--setup-session
-        ;; `org-babel-comint-wait-for-output' as python may emit
+           nil 'local)))
-        ;; multiple prompts during initialization.
+      ;; Wait until Python initializes
-        (with-current-buffer py-buffer
+      ;; This is more reliable compared to
-          (while (not org-babel-python--initialized)
+      ;; `org-babel-comint-wait-for-output' as python may emit
-            (org-babel-comint-wait-for-output py-buffer))))
+      ;; multiple prompts during initialization.
+      (with-current-buffer py-buffer
+        (while (not org-babel-python--initialized)
+          (org-babel-comint-wait-for-output py-buffer)))
       (setq org-babel-python-buffers
 	    (cons (cons session py-buffer)
 		  (assq-delete-all session org-babel-python-buffers)))
@@ -218,28 +287,6 @@ then create.  Return the initialized session."
 (defvar org-babel-python-eoe-indicator "org_babel_python_eoe"
   "A string to indicate that evaluation has completed.")
 
-(defconst org-babel-python-wrapper-method
-  "
-def main():
-%s
-
-open('%s', 'w').write( str(main()) )")
-(defconst org-babel-python-pp-wrapper-method
-  "
-import pprint
-def main():
-%s
-
-open('%s', 'w').write( pprint.pformat(main()) )")
-
-(defconst org-babel-python--exec-tmpfile "\
-with open('%s') as __org_babel_python_tmpfile:
-    exec(compile(__org_babel_python_tmpfile.read(), __org_babel_python_tmpfile.name, 'exec'))"
-  "Template for Python session command with output results.
-
-Has a single %s escape, the tempfile containing the source code
-to evaluate.")
-
 (defun org-babel-python-format-session-value
     (src-file result-file result-params)
   "Return Python code to evaluate SRC-FILE and write result to RESULT-FILE."
@@ -253,30 +300,25 @@ if isinstance(__org_babel_python_final, ast.Expr):
     exec(compile(__org_babel_python_ast, '<string>', 'exec'))
     __org_babel_python_final = eval(compile(ast.Expression(
         __org_babel_python_final.value), '<string>', 'eval'))
-    with open('%s', 'w') as __org_babel_python_tmpfile:
-        if %s:
-            import pprint
-            __org_babel_python_tmpfile.write(pprint.pformat(__org_babel_python_final))
-        else:
-            __org_babel_python_tmpfile.write(str(__org_babel_python_final))
 else:
     exec(compile(__org_babel_python_ast, '<string>', 'exec'))
-    __org_babel_python_final = None"
+    __org_babel_python_final = None
+__org_babel_python_format_value(__org_babel_python_final, '%s', %s)"
 	  (org-babel-process-file-name src-file 'noquote)
 	  (org-babel-process-file-name result-file 'noquote)
-	  (if (member "pp" result-params) "True" "False")))
+	  (org-babel-python-var-to-python result-params)))
 
 (defun org-babel-python-evaluate
-    (session body &optional result-type result-params preamble async)
+    (session body &optional result-type result-params preamble async graphics-file)
   "Evaluate BODY as Python code."
   (if session
       (if async
 	  (org-babel-python-async-evaluate-session
-	   session body result-type result-params)
+	   session body result-type result-params graphics-file)
 	(org-babel-python-evaluate-session
-	 session body result-type result-params))
+	 session body result-type result-params graphics-file))
     (org-babel-python-evaluate-external-process
-     body result-type result-params preamble)))
+     body result-type result-params preamble graphics-file)))
 
 (defun org-babel-python--shift-right (body &optional count)
   (with-temp-buffer
@@ -292,31 +334,39 @@ else:
     (buffer-string)))
 
 (defun org-babel-python-evaluate-external-process
-    (body &optional result-type result-params preamble)
+    (body &optional result-type result-params preamble graphics-file)
   "Evaluate BODY in external python process.
 If RESULT-TYPE equals `output' then return standard output as a
-string.  If RESULT-TYPE equals `value' then return the value of the
+string.  If RESULT-TYPE equals `value' then return the value of
-last statement in BODY, as elisp."
+the last statement in BODY, as elisp.  If GRAPHICS-FILE is
+non-nil, then save graphical results to that file instead."
   (let ((raw
          (pcase result-type
            (`output (org-babel-eval org-babel-python-command
 				    (concat preamble (and preamble "\n")
-					    body)))
+                                            (if graphics-file
-           (`value (let ((tmp-file (org-babel-temp-file "python-")))
+                                                (format org-babel-python--output-graphics-wrapper
+                                                        body graphics-file)
+                                              body))))
+           (`value (let ((results-file (or graphics-file
+				           (org-babel-temp-file "python-"))))
 		     (org-babel-eval
 		      org-babel-python-command
 		      (concat
 		       preamble (and preamble "\n")
 		       (format
-			(if (member "pp" result-params)
+			(concat org-babel-python--def-format-value "
-			    org-babel-python-pp-wrapper-method
+def main():
-			  org-babel-python-wrapper-method)
+%s
-			(org-babel-python--shift-right body)
+
-			(org-babel-process-file-name tmp-file 'noquote))))
+__org_babel_python_format_value(main(), '%s', %s)")
-		     (org-babel-eval-read-file tmp-file))))))
+                        (org-babel-python--shift-right body)
+			(org-babel-process-file-name results-file 'noquote)
+			(org-babel-python-var-to-python result-params))))
+		     (org-babel-eval-read-file results-file))))))
     (org-babel-result-cond result-params
       raw
-      (org-babel-python-table-or-string (org-trim raw)))))
+      (org-babel-python-table-or-string raw))))
 
 (defun org-babel-python-send-string (session body)
   "Pass BODY to the Python process in SESSION.
@@ -347,28 +397,36 @@ finally:
       (org-babel-chomp (substring string-buffer 0 (match-beginning 0))))))
 
 (defun org-babel-python-evaluate-session
-    (session body &optional result-type result-params)
+    (session body &optional result-type result-params graphics-file)
   "Pass BODY to the Python process in SESSION.
 If RESULT-TYPE equals `output' then return standard output as a
-string.  If RESULT-TYPE equals `value' then return the value of the
+string.  If RESULT-TYPE equals `value' then return the value of
-last statement in BODY, as elisp."
+the last statement in BODY, as elisp.  If GRAPHICS-FILE is
+non-nil, then save graphical results to that file instead."
   (let* ((tmp-src-file (org-babel-temp-file "python-"))
          (results
 	  (progn
-	    (with-temp-file tmp-src-file (insert body))
+	    (with-temp-file tmp-src-file
+              (insert (if (and graphics-file (eq result-type 'output))
+                          (format org-babel-python--output-graphics-wrapper
+                                  body graphics-file)
+                        body)))
             (pcase result-type
 	      (`output
-	       (let ((body (format org-babel-python--exec-tmpfile
+	       (let ((body (format "\
+with open('%s') as f:
+    exec(compile(f.read(), f.name, 'exec'))"
 				   (org-babel-process-file-name
 				    tmp-src-file 'noquote))))
 		 (org-babel-python-send-string session body)))
               (`value
-               (let* ((tmp-results-file (org-babel-temp-file "python-"))
+               (let* ((results-file (or graphics-file
+					(org-babel-temp-file "python-")))
 		      (body (org-babel-python-format-session-value
-			     tmp-src-file tmp-results-file result-params)))
+			     tmp-src-file results-file result-params)))
 		 (org-babel-python-send-string session body)
 		 (sleep-for 0 10)
-		 (org-babel-eval-read-file tmp-results-file)))))))
+		 (org-babel-eval-read-file results-file)))))))
     (org-babel-result-cond result-params
       results
       (org-babel-python-table-or-string results))))
@@ -392,7 +450,7 @@ last statement in BODY, as elisp."
       (org-babel-python-table-or-string results))))
 
 (defun org-babel-python-async-evaluate-session
-    (session body &optional result-type result-params)
+    (session body &optional result-type result-params graphics-file)
   "Asynchronously evaluate BODY in SESSION.
 Returns a placeholder string for insertion, to later be replaced
 by `org-babel-comint-async-filter'."
@@ -406,7 +464,10 @@ by `org-babel-comint-async-filter'."
        (with-temp-buffer
          (insert (format org-babel-python-async-indicator "start" uuid))
          (insert "\n")
-         (insert body)
+         (insert (if graphics-file
+                     (format org-babel-python--output-graphics-wrapper
+                             body graphics-file)
+                   body))
          (insert "\n")
          (insert (format org-babel-python-async-indicator "end" uuid))
          (let ((python-shell-buffer-name
@@ -414,17 +475,20 @@ by `org-babel-comint-async-filter'."
            (python-shell-send-buffer)))
        uuid))
     (`value
-     (let ((tmp-results-file (org-babel-temp-file "python-"))
+     (let ((results-file (or graphics-file
+			     (org-babel-temp-file "python-")))
            (tmp-src-file (org-babel-temp-file "python-")))
        (with-temp-file tmp-src-file (insert body))
        (with-temp-buffer
-         (insert (org-babel-python-format-session-value tmp-src-file tmp-results-file result-params))
+         (insert (org-babel-python-format-session-value
+                  tmp-src-file results-file result-params))
          (insert "\n")
-         (insert (format org-babel-python-async-indicator "file" tmp-results-file))
+         (unless graphics-file
+           (insert (format org-babel-python-async-indicator "file" results-file)))
          (let ((python-shell-buffer-name
                 (org-babel-python-without-earmuffs session)))
            (python-shell-send-buffer)))
-       tmp-results-file))))
+       results-file))))
 
 (provide 'ob-python)