From 579e8c572345c42ad581d3ddf0f484567d55a787 Mon Sep 17 00:00:00 2001 From: Jack Kamm Date: Mon, 7 Sep 2020 09:58:30 -0700 Subject: [PATCH] ob-python: Results handling for dicts, dataframes, arrays, plots Thanks to Ihor Radchenko and Liu Hui for valuable feedback on this patch. * lisp/ob-python.el (org-babel-execute:python): Parse graphics-file from params, and pass it to `org-babel-python-evaluate'. (org-babel-python--output-graphics-wrapper): New constant. Python code to save graphical output. (org-babel-python--def-format-value): New constant. Python function to format and save value results to file. Includes handling for graphics, dataframes, and arrays. (org-babel-python-table-or-string): Prevent `org-babel-script-escape' from mangling dict results. (org-babel-python--setup-session): New function to run setup code within Python session. (org-babel-python-initiate-session-by-key): Replace lambda with `org-babel-python--setup-session' in `python-shell-first-prompt-hook', or run it directly if the session was started outside Org. (org-babel-python-wrapper-method): Removed. Instead use part of the string directly in `org-babel-python-evaluate-external-process'. (org-babel-python-pp-wrapper-method): Removed. Pretty printing is now handled by `org-babel-python--def-format-value'. (org-babel-python--exec-tmpfile): Removed. Instead use the raw string directly in `org-babel-python-evaluate-session'. (org-babel-python-format-session-value): Updated to use `org-babel-python--def-format-value' for formatting value result. (org-babel-python-evaluate): New parameter graphics-file. Pass graphics-file onto downstream helper functions. (org-babel-python-evaluate-external-process): New parameter graphics-file. Use `org-babel-python--output-graphics-wrapper' for graphical output. For value result, use `org-babel-python--def-format-value'. Don't call `org-trim' on results, to prevent misalignment of pandas DataFrames. (org-babel-python-evaluate-session): New parameter graphics-file. Use `org-babel-python--output-graphics-wrapper' for graphical output. Replace the removed constant `org-babel-python--exec-tmpfile' with the string directly. Rename local variable tmp-results-file to results-file, which may take the value of graphics-file when provided. (org-babel-python-async-evaluate-session): New parameter graphics-file. Use `org-babel-python--output-graphics-wrapper' for graphical output. Rename local variable tmp-results-file to results-file, which may take the value of graphics-file when provided. --- etc/ORG-NEWS | 17 ++++ lisp/ob-python.el | 226 +++++++++++++++++++++++++++++----------------- 2 files changed, 162 insertions(+), 81 deletions(-) diff --git a/etc/ORG-NEWS b/etc/ORG-NEWS index 842d3fe4d..4db9d258a 100644 --- a/etc/ORG-NEWS +++ b/etc/ORG-NEWS @@ -576,6 +576,23 @@ of all relational operators (~<*~, ~=*~, ~!=*~, etc.) that work like the regular, unstarred operators but match a headline only if the tested property is actually present. +*** =ob-python.el=: Support for more result types and plotting + +=ob-python= now converts the following objects to org-mode tables when +":results table" header arg is set: + +- Dictionaries +- Numpy arrays +- Pandas DataFrames +- Pandas Series + +When the header argument =:results graphics= is set, =ob-python= will +use matplotlib to save graphics. The behavior depends on whether value +or output results are used. For value results, the last line should +return a matplotlib Figure object to plot. For output results, the +current figure (as returned by =pyplot.gcf()=) is cleared before +evaluation, and then plotted afterwards. + ** New functions and changes in function arguments *** =TYPES= argument in ~org-element-lineage~ can now be a symbol diff --git a/lisp/ob-python.el b/lisp/ob-python.el index c15d45b96..b9dab91b4 100644 --- a/lisp/ob-python.el +++ b/lisp/ob-python.el @@ -70,6 +70,8 @@ This function is called by `org-babel-execute-src-block'." org-babel-python-command)) (session (org-babel-python-initiate-session (cdr (assq :session params)))) + (graphics-file (and (member "graphics" (assq :result-params params)) + (org-babel-graphical-output-file params))) (result-params (cdr (assq :result-params params))) (result-type (cdr (assq :result-type params))) (return-val (when (eq result-type 'value) @@ -85,7 +87,7 @@ This function is called by `org-babel-execute-src-block'." (format (if session "\n%s" "\nreturn %s") return-val)))) (result (org-babel-python-evaluate session full-body result-type - result-params preamble async))) + result-params preamble async graphics-file))) (org-babel-reassemble-table result (org-babel-pick-name (cdr (assq :colname-names params)) @@ -117,6 +119,60 @@ VARS contains resolved variable references." ;; helper functions +(defconst org-babel-python--output-graphics-wrapper "\ +import matplotlib.pyplot +matplotlib.pyplot.gcf().clear() +%s +matplotlib.pyplot.savefig('%s')" + "Format string for saving Python graphical output. +Has two %s escapes, for the Python code to be evaluated, and the +file to save the graphics to.") + +(defconst org-babel-python--def-format-value "\ +def __org_babel_python_format_value(result, result_file, result_params): + with open(result_file, 'w') as f: + if 'graphics' in result_params: + result.savefig(result_file) + elif 'pp' in result_params: + import pprint + f.write(pprint.pformat(result)) + elif 'list' in result_params and isinstance(result, dict): + f.write(str(['{} :: {}'.format(k, v) for k, v in result.items()])) + else: + if not set(result_params).intersection(\ +['scalar', 'verbatim', 'raw']): + def dict2table(res): + if isinstance(res, dict): + return [(k, dict2table(v)) for k, v in res.items()] + elif isinstance(res, list) or isinstance(res, tuple): + return [dict2table(x) for x in res] + else: + return res + if 'table' in result_params: + result = dict2table(result) + try: + import pandas + except ImportError: + pass + else: + if isinstance(result, pandas.DataFrame) and 'table' in result_params: + result = [[result.index.name or ''] + list(result.columns)] + \ +[None] + [[i] + list(row) for i, row in result.iterrows()] + elif isinstance(result, pandas.Series) and 'table' in result_params: + result = list(result.items()) + try: + import numpy + except ImportError: + pass + else: + if isinstance(result, numpy.ndarray): + if 'table' in result_params: + result = result.tolist() + else: + result = repr(result) + f.write(str(result))" + "Python function to format value result and save it to file.") + (defun org-babel-variable-assignments:python (params) "Return a list of Python statements assigning the block's variables." (mapcar @@ -140,9 +196,12 @@ specifying a variable of the same value." (defun org-babel-python-table-or-string (results) "Convert RESULTS into an appropriate elisp value. -If the results look like a list or tuple, then convert them into an -Emacs-lisp table, otherwise return the results as a string." - (let ((res (org-babel-script-escape results))) +If the results look like a list or tuple (but not a dict), then +convert them into an Emacs-lisp table. Otherwise return the +results as a string." + (let ((res (if (string-equal "{" (substring results 0 1)) + results ;don't covert dicts to elisp + (org-babel-script-escape results)))) (if (listp res) (mapcar (lambda (el) (if (eq el 'None) org-babel-python-None-to el)) @@ -171,6 +230,13 @@ Emacs-lisp table, otherwise return the results as a string." (defvar-local org-babel-python--initialized nil "Flag used to mark that python session has been initialized.") +(defun org-babel-python--setup-session () + "Babel Python session setup code, to be run once per session. +Function should be run from within the Python session buffer. +This is often run as a part of `python-shell-first-prompt-hook', +unless the Python session was created outside Org." + (python-shell-send-string-no-output org-babel-python--def-format-value) + (setq-local org-babel-python--initialized t)) (defun org-babel-python-initiate-session-by-key (&optional session) "Initiate a python session. If there is not a current inferior-process-buffer in SESSION @@ -187,23 +253,26 @@ then create. Return the initialized session." (existing-session-p (comint-check-proc py-buffer))) (run-python cmd) (with-current-buffer py-buffer - ;; Adding to `python-shell-first-prompt-hook' immediately - ;; after `run-python' should be safe from race conditions, - ;; because subprocess output only arrives when Emacs is - ;; waiting (see elisp manual, "Output from Processes") - (add-hook - 'python-shell-first-prompt-hook - (lambda () (setq-local org-babel-python--initialized t)) - nil 'local)) - ;; Don't hang if session was started externally - (unless existing-session-p - ;; Wait until Python initializes - ;; This is more reliable compared to - ;; `org-babel-comint-wait-for-output' as python may emit - ;; multiple prompts during initialization. - (with-current-buffer py-buffer - (while (not org-babel-python--initialized) - (org-babel-comint-wait-for-output py-buffer)))) + (if existing-session-p + ;; Session was created outside Org. Assume first prompt + ;; already happened; run session setup code directly + (unless org-babel-python--initialized + (org-babel-python--setup-session)) + ;; Adding to `python-shell-first-prompt-hook' immediately + ;; after `run-python' should be safe from race conditions, + ;; because subprocess output only arrives when Emacs is + ;; waiting (see elisp manual, "Output from Processes") + (add-hook + 'python-shell-first-prompt-hook + #'org-babel-python--setup-session + nil 'local))) + ;; Wait until Python initializes + ;; This is more reliable compared to + ;; `org-babel-comint-wait-for-output' as python may emit + ;; multiple prompts during initialization. + (with-current-buffer py-buffer + (while (not org-babel-python--initialized) + (org-babel-comint-wait-for-output py-buffer))) (setq org-babel-python-buffers (cons (cons session py-buffer) (assq-delete-all session org-babel-python-buffers))) @@ -218,28 +287,6 @@ then create. Return the initialized session." (defvar org-babel-python-eoe-indicator "org_babel_python_eoe" "A string to indicate that evaluation has completed.") -(defconst org-babel-python-wrapper-method - " -def main(): -%s - -open('%s', 'w').write( str(main()) )") -(defconst org-babel-python-pp-wrapper-method - " -import pprint -def main(): -%s - -open('%s', 'w').write( pprint.pformat(main()) )") - -(defconst org-babel-python--exec-tmpfile "\ -with open('%s') as __org_babel_python_tmpfile: - exec(compile(__org_babel_python_tmpfile.read(), __org_babel_python_tmpfile.name, 'exec'))" - "Template for Python session command with output results. - -Has a single %s escape, the tempfile containing the source code -to evaluate.") - (defun org-babel-python-format-session-value (src-file result-file result-params) "Return Python code to evaluate SRC-FILE and write result to RESULT-FILE." @@ -253,30 +300,25 @@ if isinstance(__org_babel_python_final, ast.Expr): exec(compile(__org_babel_python_ast, '', 'exec')) __org_babel_python_final = eval(compile(ast.Expression( __org_babel_python_final.value), '', 'eval')) - with open('%s', 'w') as __org_babel_python_tmpfile: - if %s: - import pprint - __org_babel_python_tmpfile.write(pprint.pformat(__org_babel_python_final)) - else: - __org_babel_python_tmpfile.write(str(__org_babel_python_final)) else: exec(compile(__org_babel_python_ast, '', 'exec')) - __org_babel_python_final = None" + __org_babel_python_final = None +__org_babel_python_format_value(__org_babel_python_final, '%s', %s)" (org-babel-process-file-name src-file 'noquote) (org-babel-process-file-name result-file 'noquote) - (if (member "pp" result-params) "True" "False"))) + (org-babel-python-var-to-python result-params))) (defun org-babel-python-evaluate - (session body &optional result-type result-params preamble async) + (session body &optional result-type result-params preamble async graphics-file) "Evaluate BODY as Python code." (if session (if async (org-babel-python-async-evaluate-session - session body result-type result-params) + session body result-type result-params graphics-file) (org-babel-python-evaluate-session - session body result-type result-params)) + session body result-type result-params graphics-file)) (org-babel-python-evaluate-external-process - body result-type result-params preamble))) + body result-type result-params preamble graphics-file))) (defun org-babel-python--shift-right (body &optional count) (with-temp-buffer @@ -292,31 +334,39 @@ else: (buffer-string))) (defun org-babel-python-evaluate-external-process - (body &optional result-type result-params preamble) + (body &optional result-type result-params preamble graphics-file) "Evaluate BODY in external python process. If RESULT-TYPE equals `output' then return standard output as a -string. If RESULT-TYPE equals `value' then return the value of the -last statement in BODY, as elisp." +string. If RESULT-TYPE equals `value' then return the value of +the last statement in BODY, as elisp. If GRAPHICS-FILE is +non-nil, then save graphical results to that file instead." (let ((raw (pcase result-type (`output (org-babel-eval org-babel-python-command (concat preamble (and preamble "\n") - body))) - (`value (let ((tmp-file (org-babel-temp-file "python-"))) + (if graphics-file + (format org-babel-python--output-graphics-wrapper + body graphics-file) + body)))) + (`value (let ((results-file (or graphics-file + (org-babel-temp-file "python-")))) (org-babel-eval org-babel-python-command (concat preamble (and preamble "\n") (format - (if (member "pp" result-params) - org-babel-python-pp-wrapper-method - org-babel-python-wrapper-method) - (org-babel-python--shift-right body) - (org-babel-process-file-name tmp-file 'noquote)))) - (org-babel-eval-read-file tmp-file)))))) + (concat org-babel-python--def-format-value " +def main(): +%s + +__org_babel_python_format_value(main(), '%s', %s)") + (org-babel-python--shift-right body) + (org-babel-process-file-name results-file 'noquote) + (org-babel-python-var-to-python result-params)))) + (org-babel-eval-read-file results-file)))))) (org-babel-result-cond result-params raw - (org-babel-python-table-or-string (org-trim raw))))) + (org-babel-python-table-or-string raw)))) (defun org-babel-python-send-string (session body) "Pass BODY to the Python process in SESSION. @@ -347,28 +397,36 @@ finally: (org-babel-chomp (substring string-buffer 0 (match-beginning 0)))))) (defun org-babel-python-evaluate-session - (session body &optional result-type result-params) + (session body &optional result-type result-params graphics-file) "Pass BODY to the Python process in SESSION. If RESULT-TYPE equals `output' then return standard output as a -string. If RESULT-TYPE equals `value' then return the value of the -last statement in BODY, as elisp." +string. If RESULT-TYPE equals `value' then return the value of +the last statement in BODY, as elisp. If GRAPHICS-FILE is +non-nil, then save graphical results to that file instead." (let* ((tmp-src-file (org-babel-temp-file "python-")) (results (progn - (with-temp-file tmp-src-file (insert body)) + (with-temp-file tmp-src-file + (insert (if (and graphics-file (eq result-type 'output)) + (format org-babel-python--output-graphics-wrapper + body graphics-file) + body))) (pcase result-type (`output - (let ((body (format org-babel-python--exec-tmpfile + (let ((body (format "\ +with open('%s') as f: + exec(compile(f.read(), f.name, 'exec'))" (org-babel-process-file-name tmp-src-file 'noquote)))) (org-babel-python-send-string session body))) (`value - (let* ((tmp-results-file (org-babel-temp-file "python-")) + (let* ((results-file (or graphics-file + (org-babel-temp-file "python-"))) (body (org-babel-python-format-session-value - tmp-src-file tmp-results-file result-params))) + tmp-src-file results-file result-params))) (org-babel-python-send-string session body) (sleep-for 0 10) - (org-babel-eval-read-file tmp-results-file))))))) + (org-babel-eval-read-file results-file))))))) (org-babel-result-cond result-params results (org-babel-python-table-or-string results)))) @@ -392,7 +450,7 @@ last statement in BODY, as elisp." (org-babel-python-table-or-string results)))) (defun org-babel-python-async-evaluate-session - (session body &optional result-type result-params) + (session body &optional result-type result-params graphics-file) "Asynchronously evaluate BODY in SESSION. Returns a placeholder string for insertion, to later be replaced by `org-babel-comint-async-filter'." @@ -406,7 +464,10 @@ by `org-babel-comint-async-filter'." (with-temp-buffer (insert (format org-babel-python-async-indicator "start" uuid)) (insert "\n") - (insert body) + (insert (if graphics-file + (format org-babel-python--output-graphics-wrapper + body graphics-file) + body)) (insert "\n") (insert (format org-babel-python-async-indicator "end" uuid)) (let ((python-shell-buffer-name @@ -414,17 +475,20 @@ by `org-babel-comint-async-filter'." (python-shell-send-buffer))) uuid)) (`value - (let ((tmp-results-file (org-babel-temp-file "python-")) + (let ((results-file (or graphics-file + (org-babel-temp-file "python-"))) (tmp-src-file (org-babel-temp-file "python-"))) (with-temp-file tmp-src-file (insert body)) (with-temp-buffer - (insert (org-babel-python-format-session-value tmp-src-file tmp-results-file result-params)) + (insert (org-babel-python-format-session-value + tmp-src-file results-file result-params)) (insert "\n") - (insert (format org-babel-python-async-indicator "file" tmp-results-file)) + (unless graphics-file + (insert (format org-babel-python-async-indicator "file" results-file))) (let ((python-shell-buffer-name (org-babel-python-without-earmuffs session))) (python-shell-send-buffer))) - tmp-results-file)))) + results-file)))) (provide 'ob-python)