ob-python: Results handling for dicts, dataframes, arrays, plots
Thanks to Ihor Radchenko and Liu Hui for valuable feedback on this patch. * lisp/ob-python.el (org-babel-execute:python): Parse graphics-file from params, and pass it to `org-babel-python-evaluate'. (org-babel-python--output-graphics-wrapper): New constant. Python code to save graphical output. (org-babel-python--def-format-value): New constant. Python function to format and save value results to file. Includes handling for graphics, dataframes, and arrays. (org-babel-python-table-or-string): Prevent `org-babel-script-escape' from mangling dict results. (org-babel-python--setup-session): New function to run setup code within Python session. (org-babel-python-initiate-session-by-key): Replace lambda with `org-babel-python--setup-session' in `python-shell-first-prompt-hook', or run it directly if the session was started outside Org. (org-babel-python-wrapper-method): Removed. Instead use part of the string directly in `org-babel-python-evaluate-external-process'. (org-babel-python-pp-wrapper-method): Removed. Pretty printing is now handled by `org-babel-python--def-format-value'. (org-babel-python--exec-tmpfile): Removed. Instead use the raw string directly in `org-babel-python-evaluate-session'. (org-babel-python-format-session-value): Updated to use `org-babel-python--def-format-value' for formatting value result. (org-babel-python-evaluate): New parameter graphics-file. Pass graphics-file onto downstream helper functions. (org-babel-python-evaluate-external-process): New parameter graphics-file. Use `org-babel-python--output-graphics-wrapper' for graphical output. For value result, use `org-babel-python--def-format-value'. Don't call `org-trim' on results, to prevent misalignment of pandas DataFrames. (org-babel-python-evaluate-session): New parameter graphics-file. Use `org-babel-python--output-graphics-wrapper' for graphical output. Replace the removed constant `org-babel-python--exec-tmpfile' with the string directly. Rename local variable tmp-results-file to results-file, which may take the value of graphics-file when provided. (org-babel-python-async-evaluate-session): New parameter graphics-file. Use `org-babel-python--output-graphics-wrapper' for graphical output. Rename local variable tmp-results-file to results-file, which may take the value of graphics-file when provided.
This commit is contained in:
Jack Kamm
parent
53c9d91d3c
commit
579e8c5723
17
etc/ORG-NEWS
17
etc/ORG-NEWS
|
|
@ -576,6 +576,23 @@ of all relational operators (~<*~, ~=*~, ~!=*~, etc.) that work like
|
|||
the regular, unstarred operators but match a headline only if the
|
||||
tested property is actually present.
|
||||
|
||||
*** =ob-python.el=: Support for more result types and plotting
|
||||
|
||||
=ob-python= now converts the following objects to org-mode tables when
|
||||
":results table" header arg is set:
|
||||
|
||||
- Dictionaries
|
||||
- Numpy arrays
|
||||
- Pandas DataFrames
|
||||
- Pandas Series
|
||||
|
||||
When the header argument =:results graphics= is set, =ob-python= will
|
||||
use matplotlib to save graphics. The behavior depends on whether value
|
||||
or output results are used. For value results, the last line should
|
||||
return a matplotlib Figure object to plot. For output results, the
|
||||
current figure (as returned by =pyplot.gcf()=) is cleared before
|
||||
evaluation, and then plotted afterwards.
|
||||
|
||||
** New functions and changes in function arguments
|
||||
*** =TYPES= argument in ~org-element-lineage~ can now be a symbol
|
||||
|
||||
|
|
|
|||
|
|
@ -70,6 +70,8 @@ This function is called by `org-babel-execute-src-block'."
|
|||
org-babel-python-command))
|
||||
(session (org-babel-python-initiate-session
|
||||
(cdr (assq :session params))))
|
||||
(graphics-file (and (member "graphics" (assq :result-params params))
|
||||
(org-babel-graphical-output-file params)))
|
||||
(result-params (cdr (assq :result-params params)))
|
||||
(result-type (cdr (assq :result-type params)))
|
||||
(return-val (when (eq result-type 'value)
|
||||
|
|
@ -85,7 +87,7 @@ This function is called by `org-babel-execute-src-block'."
|
|||
(format (if session "\n%s" "\nreturn %s") return-val))))
|
||||
(result (org-babel-python-evaluate
|
||||
session full-body result-type
|
||||
result-params preamble async)))
|
||||
result-params preamble async graphics-file)))
|
||||
(org-babel-reassemble-table
|
||||
result
|
||||
(org-babel-pick-name (cdr (assq :colname-names params))
|
||||
|
|
@ -117,6 +119,60 @@ VARS contains resolved variable references."
|
|||
|
||||
;; helper functions
|
||||
|
||||
(defconst org-babel-python--output-graphics-wrapper "\
|
||||
import matplotlib.pyplot
|
||||
matplotlib.pyplot.gcf().clear()
|
||||
%s
|
||||
matplotlib.pyplot.savefig('%s')"
|
||||
"Format string for saving Python graphical output.
|
||||
Has two %s escapes, for the Python code to be evaluated, and the
|
||||
file to save the graphics to.")
|
||||
|
||||
(defconst org-babel-python--def-format-value "\
|
||||
def __org_babel_python_format_value(result, result_file, result_params):
|
||||
with open(result_file, 'w') as f:
|
||||
if 'graphics' in result_params:
|
||||
result.savefig(result_file)
|
||||
elif 'pp' in result_params:
|
||||
import pprint
|
||||
f.write(pprint.pformat(result))
|
||||
elif 'list' in result_params and isinstance(result, dict):
|
||||
f.write(str(['{} :: {}'.format(k, v) for k, v in result.items()]))
|
||||
else:
|
||||
if not set(result_params).intersection(\
|
||||
['scalar', 'verbatim', 'raw']):
|
||||
def dict2table(res):
|
||||
if isinstance(res, dict):
|
||||
return [(k, dict2table(v)) for k, v in res.items()]
|
||||
elif isinstance(res, list) or isinstance(res, tuple):
|
||||
return [dict2table(x) for x in res]
|
||||
else:
|
||||
return res
|
||||
if 'table' in result_params:
|
||||
result = dict2table(result)
|
||||
try:
|
||||
import pandas
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
if isinstance(result, pandas.DataFrame) and 'table' in result_params:
|
||||
result = [[result.index.name or ''] + list(result.columns)] + \
|
||||
[None] + [[i] + list(row) for i, row in result.iterrows()]
|
||||
elif isinstance(result, pandas.Series) and 'table' in result_params:
|
||||
result = list(result.items())
|
||||
try:
|
||||
import numpy
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
if isinstance(result, numpy.ndarray):
|
||||
if 'table' in result_params:
|
||||
result = result.tolist()
|
||||
else:
|
||||
result = repr(result)
|
||||
f.write(str(result))"
|
||||
"Python function to format value result and save it to file.")
|
||||
|
||||
(defun org-babel-variable-assignments:python (params)
|
||||
"Return a list of Python statements assigning the block's variables."
|
||||
(mapcar
|
||||
|
|
@ -140,9 +196,12 @@ specifying a variable of the same value."
|
|||
|
||||
(defun org-babel-python-table-or-string (results)
|
||||
"Convert RESULTS into an appropriate elisp value.
|
||||
If the results look like a list or tuple, then convert them into an
|
||||
Emacs-lisp table, otherwise return the results as a string."
|
||||
(let ((res (org-babel-script-escape results)))
|
||||
If the results look like a list or tuple (but not a dict), then
|
||||
convert them into an Emacs-lisp table. Otherwise return the
|
||||
results as a string."
|
||||
(let ((res (if (string-equal "{" (substring results 0 1))
|
||||
results ;don't covert dicts to elisp
|
||||
(org-babel-script-escape results))))
|
||||
(if (listp res)
|
||||
(mapcar (lambda (el) (if (eq el 'None)
|
||||
org-babel-python-None-to el))
|
||||
|
|
@ -171,6 +230,13 @@ Emacs-lisp table, otherwise return the results as a string."
|
|||
|
||||
(defvar-local org-babel-python--initialized nil
|
||||
"Flag used to mark that python session has been initialized.")
|
||||
(defun org-babel-python--setup-session ()
|
||||
"Babel Python session setup code, to be run once per session.
|
||||
Function should be run from within the Python session buffer.
|
||||
This is often run as a part of `python-shell-first-prompt-hook',
|
||||
unless the Python session was created outside Org."
|
||||
(python-shell-send-string-no-output org-babel-python--def-format-value)
|
||||
(setq-local org-babel-python--initialized t))
|
||||
(defun org-babel-python-initiate-session-by-key (&optional session)
|
||||
"Initiate a python session.
|
||||
If there is not a current inferior-process-buffer in SESSION
|
||||
|
|
@ -187,23 +253,26 @@ then create. Return the initialized session."
|
|||
(existing-session-p (comint-check-proc py-buffer)))
|
||||
(run-python cmd)
|
||||
(with-current-buffer py-buffer
|
||||
;; Adding to `python-shell-first-prompt-hook' immediately
|
||||
;; after `run-python' should be safe from race conditions,
|
||||
;; because subprocess output only arrives when Emacs is
|
||||
;; waiting (see elisp manual, "Output from Processes")
|
||||
(add-hook
|
||||
'python-shell-first-prompt-hook
|
||||
(lambda () (setq-local org-babel-python--initialized t))
|
||||
nil 'local))
|
||||
;; Don't hang if session was started externally
|
||||
(unless existing-session-p
|
||||
;; Wait until Python initializes
|
||||
;; This is more reliable compared to
|
||||
;; `org-babel-comint-wait-for-output' as python may emit
|
||||
;; multiple prompts during initialization.
|
||||
(with-current-buffer py-buffer
|
||||
(while (not org-babel-python--initialized)
|
||||
(org-babel-comint-wait-for-output py-buffer))))
|
||||
(if existing-session-p
|
||||
;; Session was created outside Org. Assume first prompt
|
||||
;; already happened; run session setup code directly
|
||||
(unless org-babel-python--initialized
|
||||
(org-babel-python--setup-session))
|
||||
;; Adding to `python-shell-first-prompt-hook' immediately
|
||||
;; after `run-python' should be safe from race conditions,
|
||||
;; because subprocess output only arrives when Emacs is
|
||||
;; waiting (see elisp manual, "Output from Processes")
|
||||
(add-hook
|
||||
'python-shell-first-prompt-hook
|
||||
#'org-babel-python--setup-session
|
||||
nil 'local)))
|
||||
;; Wait until Python initializes
|
||||
;; This is more reliable compared to
|
||||
;; `org-babel-comint-wait-for-output' as python may emit
|
||||
;; multiple prompts during initialization.
|
||||
(with-current-buffer py-buffer
|
||||
(while (not org-babel-python--initialized)
|
||||
(org-babel-comint-wait-for-output py-buffer)))
|
||||
(setq org-babel-python-buffers
|
||||
(cons (cons session py-buffer)
|
||||
(assq-delete-all session org-babel-python-buffers)))
|
||||
|
|
@ -218,28 +287,6 @@ then create. Return the initialized session."
|
|||
(defvar org-babel-python-eoe-indicator "org_babel_python_eoe"
|
||||
"A string to indicate that evaluation has completed.")
|
||||
|
||||
(defconst org-babel-python-wrapper-method
|
||||
"
|
||||
def main():
|
||||
%s
|
||||
|
||||
open('%s', 'w').write( str(main()) )")
|
||||
(defconst org-babel-python-pp-wrapper-method
|
||||
"
|
||||
import pprint
|
||||
def main():
|
||||
%s
|
||||
|
||||
open('%s', 'w').write( pprint.pformat(main()) )")
|
||||
|
||||
(defconst org-babel-python--exec-tmpfile "\
|
||||
with open('%s') as __org_babel_python_tmpfile:
|
||||
exec(compile(__org_babel_python_tmpfile.read(), __org_babel_python_tmpfile.name, 'exec'))"
|
||||
"Template for Python session command with output results.
|
||||
|
||||
Has a single %s escape, the tempfile containing the source code
|
||||
to evaluate.")
|
||||
|
||||
(defun org-babel-python-format-session-value
|
||||
(src-file result-file result-params)
|
||||
"Return Python code to evaluate SRC-FILE and write result to RESULT-FILE."
|
||||
|
|
@ -253,30 +300,25 @@ if isinstance(__org_babel_python_final, ast.Expr):
|
|||
exec(compile(__org_babel_python_ast, '<string>', 'exec'))
|
||||
__org_babel_python_final = eval(compile(ast.Expression(
|
||||
__org_babel_python_final.value), '<string>', 'eval'))
|
||||
with open('%s', 'w') as __org_babel_python_tmpfile:
|
||||
if %s:
|
||||
import pprint
|
||||
__org_babel_python_tmpfile.write(pprint.pformat(__org_babel_python_final))
|
||||
else:
|
||||
__org_babel_python_tmpfile.write(str(__org_babel_python_final))
|
||||
else:
|
||||
exec(compile(__org_babel_python_ast, '<string>', 'exec'))
|
||||
__org_babel_python_final = None"
|
||||
__org_babel_python_final = None
|
||||
__org_babel_python_format_value(__org_babel_python_final, '%s', %s)"
|
||||
(org-babel-process-file-name src-file 'noquote)
|
||||
(org-babel-process-file-name result-file 'noquote)
|
||||
(if (member "pp" result-params) "True" "False")))
|
||||
(org-babel-python-var-to-python result-params)))
|
||||
|
||||
(defun org-babel-python-evaluate
|
||||
(session body &optional result-type result-params preamble async)
|
||||
(session body &optional result-type result-params preamble async graphics-file)
|
||||
"Evaluate BODY as Python code."
|
||||
(if session
|
||||
(if async
|
||||
(org-babel-python-async-evaluate-session
|
||||
session body result-type result-params)
|
||||
session body result-type result-params graphics-file)
|
||||
(org-babel-python-evaluate-session
|
||||
session body result-type result-params))
|
||||
session body result-type result-params graphics-file))
|
||||
(org-babel-python-evaluate-external-process
|
||||
body result-type result-params preamble)))
|
||||
body result-type result-params preamble graphics-file)))
|
||||
|
||||
(defun org-babel-python--shift-right (body &optional count)
|
||||
(with-temp-buffer
|
||||
|
|
@ -292,31 +334,39 @@ else:
|
|||
(buffer-string)))
|
||||
|
||||
(defun org-babel-python-evaluate-external-process
|
||||
(body &optional result-type result-params preamble)
|
||||
(body &optional result-type result-params preamble graphics-file)
|
||||
"Evaluate BODY in external python process.
|
||||
If RESULT-TYPE equals `output' then return standard output as a
|
||||
string. If RESULT-TYPE equals `value' then return the value of the
|
||||
last statement in BODY, as elisp."
|
||||
string. If RESULT-TYPE equals `value' then return the value of
|
||||
the last statement in BODY, as elisp. If GRAPHICS-FILE is
|
||||
non-nil, then save graphical results to that file instead."
|
||||
(let ((raw
|
||||
(pcase result-type
|
||||
(`output (org-babel-eval org-babel-python-command
|
||||
(concat preamble (and preamble "\n")
|
||||
body)))
|
||||
(`value (let ((tmp-file (org-babel-temp-file "python-")))
|
||||
(if graphics-file
|
||||
(format org-babel-python--output-graphics-wrapper
|
||||
body graphics-file)
|
||||
body))))
|
||||
(`value (let ((results-file (or graphics-file
|
||||
(org-babel-temp-file "python-"))))
|
||||
(org-babel-eval
|
||||
org-babel-python-command
|
||||
(concat
|
||||
preamble (and preamble "\n")
|
||||
(format
|
||||
(if (member "pp" result-params)
|
||||
org-babel-python-pp-wrapper-method
|
||||
org-babel-python-wrapper-method)
|
||||
(org-babel-python--shift-right body)
|
||||
(org-babel-process-file-name tmp-file 'noquote))))
|
||||
(org-babel-eval-read-file tmp-file))))))
|
||||
(concat org-babel-python--def-format-value "
|
||||
def main():
|
||||
%s
|
||||
|
||||
__org_babel_python_format_value(main(), '%s', %s)")
|
||||
(org-babel-python--shift-right body)
|
||||
(org-babel-process-file-name results-file 'noquote)
|
||||
(org-babel-python-var-to-python result-params))))
|
||||
(org-babel-eval-read-file results-file))))))
|
||||
(org-babel-result-cond result-params
|
||||
raw
|
||||
(org-babel-python-table-or-string (org-trim raw)))))
|
||||
(org-babel-python-table-or-string raw))))
|
||||
|
||||
(defun org-babel-python-send-string (session body)
|
||||
"Pass BODY to the Python process in SESSION.
|
||||
|
|
@ -347,28 +397,36 @@ finally:
|
|||
(org-babel-chomp (substring string-buffer 0 (match-beginning 0))))))
|
||||
|
||||
(defun org-babel-python-evaluate-session
|
||||
(session body &optional result-type result-params)
|
||||
(session body &optional result-type result-params graphics-file)
|
||||
"Pass BODY to the Python process in SESSION.
|
||||
If RESULT-TYPE equals `output' then return standard output as a
|
||||
string. If RESULT-TYPE equals `value' then return the value of the
|
||||
last statement in BODY, as elisp."
|
||||
string. If RESULT-TYPE equals `value' then return the value of
|
||||
the last statement in BODY, as elisp. If GRAPHICS-FILE is
|
||||
non-nil, then save graphical results to that file instead."
|
||||
(let* ((tmp-src-file (org-babel-temp-file "python-"))
|
||||
(results
|
||||
(progn
|
||||
(with-temp-file tmp-src-file (insert body))
|
||||
(with-temp-file tmp-src-file
|
||||
(insert (if (and graphics-file (eq result-type 'output))
|
||||
(format org-babel-python--output-graphics-wrapper
|
||||
body graphics-file)
|
||||
body)))
|
||||
(pcase result-type
|
||||
(`output
|
||||
(let ((body (format org-babel-python--exec-tmpfile
|
||||
(let ((body (format "\
|
||||
with open('%s') as f:
|
||||
exec(compile(f.read(), f.name, 'exec'))"
|
||||
(org-babel-process-file-name
|
||||
tmp-src-file 'noquote))))
|
||||
(org-babel-python-send-string session body)))
|
||||
(`value
|
||||
(let* ((tmp-results-file (org-babel-temp-file "python-"))
|
||||
(let* ((results-file (or graphics-file
|
||||
(org-babel-temp-file "python-")))
|
||||
(body (org-babel-python-format-session-value
|
||||
tmp-src-file tmp-results-file result-params)))
|
||||
tmp-src-file results-file result-params)))
|
||||
(org-babel-python-send-string session body)
|
||||
(sleep-for 0 10)
|
||||
(org-babel-eval-read-file tmp-results-file)))))))
|
||||
(org-babel-eval-read-file results-file)))))))
|
||||
(org-babel-result-cond result-params
|
||||
results
|
||||
(org-babel-python-table-or-string results))))
|
||||
|
|
@ -392,7 +450,7 @@ last statement in BODY, as elisp."
|
|||
(org-babel-python-table-or-string results))))
|
||||
|
||||
(defun org-babel-python-async-evaluate-session
|
||||
(session body &optional result-type result-params)
|
||||
(session body &optional result-type result-params graphics-file)
|
||||
"Asynchronously evaluate BODY in SESSION.
|
||||
Returns a placeholder string for insertion, to later be replaced
|
||||
by `org-babel-comint-async-filter'."
|
||||
|
|
@ -406,7 +464,10 @@ by `org-babel-comint-async-filter'."
|
|||
(with-temp-buffer
|
||||
(insert (format org-babel-python-async-indicator "start" uuid))
|
||||
(insert "\n")
|
||||
(insert body)
|
||||
(insert (if graphics-file
|
||||
(format org-babel-python--output-graphics-wrapper
|
||||
body graphics-file)
|
||||
body))
|
||||
(insert "\n")
|
||||
(insert (format org-babel-python-async-indicator "end" uuid))
|
||||
(let ((python-shell-buffer-name
|
||||
|
|
@ -414,17 +475,20 @@ by `org-babel-comint-async-filter'."
|
|||
(python-shell-send-buffer)))
|
||||
uuid))
|
||||
(`value
|
||||
(let ((tmp-results-file (org-babel-temp-file "python-"))
|
||||
(let ((results-file (or graphics-file
|
||||
(org-babel-temp-file "python-")))
|
||||
(tmp-src-file (org-babel-temp-file "python-")))
|
||||
(with-temp-file tmp-src-file (insert body))
|
||||
(with-temp-buffer
|
||||
(insert (org-babel-python-format-session-value tmp-src-file tmp-results-file result-params))
|
||||
(insert (org-babel-python-format-session-value
|
||||
tmp-src-file results-file result-params))
|
||||
(insert "\n")
|
||||
(insert (format org-babel-python-async-indicator "file" tmp-results-file))
|
||||
(unless graphics-file
|
||||
(insert (format org-babel-python-async-indicator "file" results-file)))
|
||||
(let ((python-shell-buffer-name
|
||||
(org-babel-python-without-earmuffs session)))
|
||||
(python-shell-send-buffer)))
|
||||
tmp-results-file))))
|
||||
results-file))))
|
||||
|
||||
(provide 'ob-python)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue