From f6452b272a6a69d3a42690a5a6b338e012428645 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Sat, 6 Jun 2009 18:06:45 -0400 Subject: [PATCH 1/7] Removing all support for hlines/colnames -- they persist in branch rowcolnames. Also removing comments. --- lisp/org-babel-R.el | 40 +++++++--------------------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/lisp/org-babel-R.el b/lisp/org-babel-R.el index a48ed25f0..6f9525657 100644 --- a/lisp/org-babel-R.el +++ b/lisp/org-babel-R.el @@ -62,16 +62,14 @@ R process in `org-babel-R-buffer'." (unless org-babel-R-buffer (error "No active R buffer")) (org-babel-R-input-command (if (listp value) - (let ((transition-file (make-temp-file "org-babel-R-import")) - has-header) + (let ((transition-file (make-temp-file "org-babel-R-import"))) ;; ensure VALUE has an orgtbl structure (depth of at least 2) (unless (listp (car value)) (setq value (list value))) - (setq has-header (and (symbolp (cadr value)) (equal (cadr value) 'hline))) (with-temp-file transition-file (insert (orgtbl-to-tsv value '(:fmt org-babel-R-quote-tsv-field))) (insert "\n")) - (format "%s <- read.table(\"%s\", header=%s, sep=\"\\t\", as.is=TRUE)" - name transition-file (if has-header "TRUE" "FALSE"))) + (format "%s <- read.table(\"%s\", header=FALSE, sep=\"\\t\", as.is=TRUE)" + name transition-file)) (format "%s <- %s" name (org-babel-R-quote-tsv-field value))))) (defun org-babel-R-to-elisp (func-name) @@ -79,8 +77,9 @@ R process in `org-babel-R-buffer'." `org-babel-R-buffer' as Emacs lisp." (let ((tmp-file (make-temp-file "org-babel-R")) result) (org-babel-R-input-command - (format "write.table(%s(), file=\"%s\", sep=\"\\t\", na=\"nil\",row.names=FALSE, col.names=FALSE, quote=FALSE)" - func-name tmp-file)) + (format + "write.table(%s(), file=\"%s\", sep=\"\\t\", na=\"nil\",row.names=FALSE, col.names=FALSE, quote=FALSE)" + func-name tmp-file)) (with-temp-buffer (condition-case nil (progn @@ -88,9 +87,7 @@ R process in `org-babel-R-buffer'." (delete-file tmp-file) (setq result (mapcar (lambda (row) (mapcar #'org-babel-R-read row)) - (org-table-to-lisp))) - ;; (setq result (org-babel-R-set-header-row result)) - ) + (org-table-to-lisp)))) (error nil)) (if (null (cdr result)) ;; if result is trivial vector, then scalarize it (if (consp (car result)) @@ -100,27 +97,6 @@ R process in `org-babel-R-buffer'." (car result)) result)))) -(defun org-babel-R-set-header-row (table) - "Check whether the table appears to have (a) genuine -user-supplied column names, or (b) default column names added -automatically by R. In case (a), maintain the first row of the -table as a header row and insert an hline. In case (b), remove -the first row and return the org table without an hline." - (if (or (string-equal (caar table) "V1") - (string-equal (caar table) "x")) - - ;; write.table(1, col.names=TRUE) makes a colname called "x". I - ;; think shows that this approach is too much of a hack: we - ;; can't take some totally different action just because we see - ;; an "x" there that might or might not be a automatic name. - - ;; The first row looks like it contains default column names - ;; added by R. This condition could be improved so that it - ;; checks whether the first row is ("V1" "V2" ... "V$n") where - ;; $n is the number of columns. - (cdr table) - (cons (car table) (cons 'hline (cdr table))))) - (defun org-babel-R-read (cell) "Strip nested \"s from around strings in exported R values." (org-babel-read (or (and (stringp cell) @@ -134,8 +110,6 @@ the first row and return the org table without an hline." (defun org-babel-R-initiate-R-buffer () "If there is not a current R process then create one." - ;; DED: Ideally I think we should use ESS mechanisms for this sort - ;; of thing. See ess-force-buffer-current. (unless (and (buffer-live-p org-babel-R-buffer) (get-buffer org-babel-R-buffer)) (save-excursion (R) From f49ac94e6a809daf70e583eeaa2394a2e0389bfd Mon Sep 17 00:00:00 2001 From: Eric Schulte Date: Mon, 8 Jun 2009 15:42:47 -0700 Subject: [PATCH 2/7] pulled org-babel.org from session branch --- org-babel.org | 251 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 246 insertions(+), 5 deletions(-) diff --git a/org-babel.org b/org-babel.org index ad7ec721a..5bcf4fb08 100644 --- a/org-babel.org +++ b/org-babel.org @@ -1,7 +1,7 @@ #+OPTIONS: H:3 num:nil toc:t #+TITLE: org-babel --- facilitating communication between programming languages and people #+SEQ_TODO: TODO PROPOSED | DONE DEFERRED REJECTED -#+STARTUP: oddeven +#+STARTUP: oddeven hideblocks * Introduction @@ -114,8 +114,9 @@ table, allowing the test suite to be run be evaluation of the table and the results to be collected in the same table. -* Tasks [22/37] -** TODO Create objects in top level (global) environment in R? [0/5] +* Tasks [22/38] +** TODO Create objects in top level (global) environment [0/5] +*sessions* *** initial requirement statement [DED] At the moment, objects created by computations performed in the @@ -189,6 +190,222 @@ or if you think any parts might be confusing for people coming from Sweave. I'll hopefully find some time to work on this later in the week. +*** can functional and interpreted/interactive models coexist? + +Even though both of these use the same =*R*= buffer the value of =a= +is not preserved because it is assigned inside of a functional +wrapper. + +#+srcname: task-R-sessions +#+begin_src R +a <- 9 +b <- 21 +a + b +#+end_src + +#+srcname: task-R-same-session +#+begin_src R +a +#+end_src + +This functional wrapper was implemented in order to efficiently return +the results of the execution of the entire source code block. However +it inhibits the evaluation of source code blocks in the top level, +which would allow for persistence of variable assignment across +evaluations. How can we allow *both* evaluation in the top level, and +efficient capture of the return value of an entire source code block +in a language independent manner? + +Possible solutions... +1) we can't so we will have to implement two types of evaluation + depending on which is appropriate (functional or imperative) +2) we remove the functional wrapper and parse the source code block + into it's top level statements (most often but not always on line + breaks) so that we can isolate the final segment which is our + return value. +3) we add some sort of "#+return" line to the code block +4) we take advantage of each languages support for meta-programming + through =eval= type functions, and use said to evaluate the entire + blocks in such a way that their environment can be combined with the + global environment, and their results are still captured. +5) I believe that most modern languages which support interactive + sessions have support for a =last_result= type function, which + returns the result of the last input without re-calculation. If + widely enough present this would be the ideal solution to a + combination of functional and imperative styles. + +None of these solutions seem very desirable, but for now I don't see +what else would be possible. + +Of these options I was leaning towards (1) and (4) but now believe +that if it is possible option (5) will be ideal. + +**** (1) both functional and imperative evaluation +Pros +- can take advantage of built in functions for sending regions to the + inferior process +- retains the proven tested and working functional wrappers + +Cons +- introduces the complication of keeping track of which type of + evaluation is best suited to a particular context +- the current functional wrappers may require some changes in order to + include the existing global context + +**** (4) exploit language meta-programming constructs to explicitly evaluate code +Pros +- only one type of evaluation + +Cons +- some languages may not have sufficient meta-programming constructs + +**** (5) exploit some =last_value= functionality if present + +Need to ensure that most languages have such a function, those without +will simply have to implement their own similar solution... + +| language | =last_value= function | +|------------+-----------------------------| +| R | .Last.value | +| ruby | _ | +| python | _ | +| shell | see [[* last command for shells][last command for shells]] | +| emacs-lisp | see [[* emacs-lisp will be a special case][special-case]] | + +#+srcname: task-last-value +#+begin_src ruby +82 + 18 +#+end_src + +***** last command for shells +Do this using the =tee= shell command, and continually pipe the output +to a file. + +Got this idea from the following [[http://linux.derkeiler.com/Mailing-Lists/Fedora/2004-01/0898.html][email-thread]]. + +suggested from mailing list + +#+srcname: bash-save-last-output-to-file +#+begin_src sh +while read line +do + bash -c "$line" | tee /tmp/last.out1 + mv /tmp/last.out1 /tmp/last.out +done +#+end_src + +another proposed solution from the above thread + +#+srcname: bash-save-in-variable +#+begin_src sh +#!/bin/bash +# so - Save Output. Saves output of command in OUT shell variable. +OUT=`$*` +echo $OUT +#+end_src + +and another + +#+begin_quote +.inputrc: +"^[k": accept-line +"^M": " | tee /tmp/h_lastcmd.out ^[k" + +.bash_profile: +export __=/tmp/h_lastcmd.out + +If you try it, Alt-k will stand for the old Enter; use "command $__" to +access the last output. + +Best, + +-- + +Herculano de Lima Einloft Neto +#+end_quote + +***** emacs-lisp will be a special case +While it is possible for emacs-lisp to be run in a console type +environment (see the =elim= function) it is *not* possible to run +emacs-lisp in a different *session*. Meaning any variable set top +level of the console environment will be set *everywhere* inside +emacs. For this reason I think that it doesn't make any sense to +worry about session support for emacs-lisp. + +*** Further thoughts on 'scripting' vs. functional approaches + + These are just thoughts, I don't know how sure I am about this. + And again, perhaps I'm not saying anything very radical, just that + it would be nice to have some options supporting things like + receiving text output in the org buffer. + + I can see that you've already gone some way down the road towards + the 'last value' approach, so sorry if my comments come rather + late. I am concerned that we are not giving sufficient attention + to stdout / the text that is returned by the interpreters. In + contrast, many of our potential users will be accustomed to a + 'scripting' approach, where they are outputting text at various + points in the code block, not just at the end. I am leaning + towards thinking that we should have 2 modes of evaluation: + 'script' mode, and 'functional' mode. + + In script mode, evaluation of a code block would result in *all* + text output from that code block appearing as output in the org + buffer, presumably as an #+begin_example...#+end_example. There + could be an :echo option controlling whether the input commands + also appear in the output. [This is like Sweave]. + + In functional mode, the *result* of the code block is available as + an elisp object, and may appear in the org buffer as an org + table/string, via the mechanisms you have developed already. + + One thing I'm wondering about is whether, in script mode, there + simply should not be a return value. Perhaps this is not so + different from what exists: script mode would be new, and what + exists currently would be functional mode. + + I think it's likely that, while code evaluation will be exciting + to people, a large majority of our users in a large majority of + their usage will not attempt to actually use the return value from + a source code block in any meaningful way. In that case, it seems + rather restrictive to only allow them to see output from the end + of the code block. + + Instead I think the most accessible way to introduce org-babel to + people, at least while they are learning it, is as an immensely + powerful environment in which to embed their 'scripts', which now + also allows them to 'run' their 'scripts'. Especially as such + people are likely to be the least capable of the user-base, a + possible design-rule would be to make the scripting style of usage + easy (default?), perhaps requiring a special option to enable a + functional style. Those who will use the functional style won't + have a problem understanding what's going on, whereas the 'skript + kiddies' might not even know the syntax for defining a function in + their language of choice. And of course we can allow the user to + set a variable in their .emacs controlling the preference, so that + functional users are not inconveniennced by having to provide + header args the whole time. + + Please don't get the impression that I am down-valuing the + functional style of org-babel. I am constantly horrified at the + messy 'scripts' that my colleagues produce in perl or R or + whatever! Nevertheless that seems to be how a lot of people work. + + I think you were leaning towards the last-value approach because + it offered the possibility of unified code supporting both the + single evaluation environment and the functional style. If you + agree with any of the above then perhaps it will impact upon this + and mean that the code in the two branches has to differ a bit. In + that case, functional mode could perhaps after all evaluate each + code block in its own environment, thus (re)approaching 'true' + functional programming (side-effects are hard to achieve). + +#+begin_src sh +ls > files +echo "There are `wc -l files` files in this directory" + +#+end_src + *** TODO rework all source codes to use inferior-processes-buffers this will involve... @@ -198,6 +415,27 @@ this will involve... 3) functions for retrieving results from the *sessions* buffers which can be overridden by each source code +**** DONE R + +#+srcname: task-R-with-inf-process-buffer +#+begin_src R +a <- 8 +b <- 9 +c <- 10 +a + b +#+end_src + +**** TODO Ruby + +#+srcname: ruby-use-last-output +#+begin_src ruby +a = 1 +b = 2 +c = 3 +(a + b) * c +#+end_src + + *** TODO implement a *session* header argument use this header argument to override the default *session* buffer @@ -284,7 +522,6 @@ org-mode core This should be implemented in the org-mode core - *** DEFERRED send code to inferior process Another thought on this topic: I think we will want users to send chunks of code to the interpreter from within the *Org Edit Src* @@ -347,6 +584,11 @@ waiting for guidance from those more familiar with yasnippets - [[file:lisp/org-babel-ref.el::TODO%20allow%20searching%20for%20names%20in%20other%20buffers][org-babel-ref.el:searching-in-other-buffers]] - [[file:lisp/org-babel.el::defun%20org-babel%20find%20named%20result%20name][org-babel.el#org-babel-find-named-result]] +** TODO figure out how to handle errors during evaluation + R has a try function, with error handling, along the lines of + python. I bet ruby does too. Maybe more of an issue for functional + style; in my proposed scripting style the error just gets dumped to + the org buffer and the user is thus alerted. ** TODO figure out how to handle graphic output This is listed under [[* graphical output][graphical output]] in out objectives. @@ -545,7 +787,6 @@ msg + "_y_python" (concat msg "_elisp") #+end_src - ** PROPOSED conversion between org-babel and noweb (e.g. .Rnw) format I haven't thought about this properly. Just noting it down. What Sweave uses is called "R noweb" (.Rnw). From 36480d34b91c9a3061cc3df197be109a86458557 Mon Sep 17 00:00:00 2001 From: Eric Schulte Date: Mon, 8 Jun 2009 16:31:00 -0700 Subject: [PATCH 3/7] even more thoughts on evaluation, results, models and options --- org-babel.org | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/org-babel.org b/org-babel.org index 5bcf4fb08..864f4964f 100644 --- a/org-babel.org +++ b/org-babel.org @@ -406,6 +406,50 @@ echo "There are `wc -l files` files in this directory" #+end_src +*** even more thoughts on evaluation, results, models and options + +Thanks Dan, These comments are invaluable. + +What do you think about this as a new list of priorities/requirements +for the execution of source-code blocks. + +1) Sessions + 1) we want the evaluation of the source code block to take place in a + session which can persist state (variables, current directory, + etc...). + 2) source code blocks can specify their session with a header argument + 3) each session should correspond to an Emacs comint buffer so that the + user can drop into the session and experiment with live code + evaluation. +2) Results + 1) each source-code block generates some form of results which (as + we have already implemented) is transfered into emacs-lisp + after which it can be inserted into the org-mode buffer, or + used by other source-code blocks + 2) when the results are translated into emacs-lisp, forced to be + interpreted as a scalar (dumping their raw values into the + org-mode buffer), as a vector (which is often desirable with R + code blocks), or interpreted on the fly (the default option). + Note that this is very nearly currently implemented through the + [[* DONE results-type header (vector/file)][results-type-header]]. + 3) there should be *two* means of collecting results from the + execution of a source code block. *Either* the value of the + last statement of the source code block, or the collection of + all that has been passed to STDOUT during the evaluation. + +**** header argument or return line + + + rather than using a header argument to specify how the return value + should be passed back, I'm leaning towards the use of a =#+RETURN= + line inside the block. If such a line *is not present* then we + default to using STDOUT to collect results, but if such a line *is + present* then we use it's value as the results of the block. I + think this will allow for the most elegant specification between + functional and script execution. This also cleans up some issues + of implementation and finding which statement is the last + statement. + *** TODO rework all source codes to use inferior-processes-buffers this will involve... @@ -435,7 +479,6 @@ c = 3 (a + b) * c #+end_src - *** TODO implement a *session* header argument use this header argument to override the default *session* buffer From 78a0f21f6dcc68e032df14f5b80a4cf11b396e12 Mon Sep 17 00:00:00 2001 From: Eric Schulte Date: Mon, 8 Jun 2009 16:38:13 -0700 Subject: [PATCH 4/7] tweaking tasks for sessions and script/functional evaluation --- org-babel.org | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/org-babel.org b/org-babel.org index 864f4964f..414016fe6 100644 --- a/org-babel.org +++ b/org-babel.org @@ -115,7 +115,7 @@ and the results to be collected in the same table. * Tasks [22/38] -** TODO Create objects in top level (global) environment [0/5] +** TODO Create objects in top level (global) environment [0/6] *sessions* *** initial requirement statement [DED] @@ -450,16 +450,13 @@ for the execution of source-code blocks. of implementation and finding which statement is the last statement. -*** TODO rework all source codes to use inferior-processes-buffers +*** TODO rework evaluation lang-by-lang [0/4] -this will involve... -1) creating a a-list of default *session* buffers for each source language -2) functions for dumping code to the *session* buffers which can be - overridden by each source code language -3) functions for retrieving results from the *sessions* buffers which - can be overridden by each source code +This should include... +- STDOUT and functional results +- sessions in comint buffers -**** DONE R +**** TODO R #+srcname: task-R-with-inf-process-buffer #+begin_src R @@ -479,15 +476,14 @@ c = 3 (a + b) * c #+end_src +**** TODO Python + +**** TODO Shells + *** TODO implement a *session* header argument use this header argument to override the default *session* buffer -*** TODO remove source bodies from their functional wrappers - -The current functional wrappers should be removed in favor of -incremental evaluation in inferior-source-buffers - *** TODO function to bring up inferior-process buffer This should be callable from inside of a source-code block in an From 29d026816f53849c9a305eae642d947c427c3f2b Mon Sep 17 00:00:00 2001 From: Eric Schulte Date: Mon, 8 Jun 2009 17:58:53 -0700 Subject: [PATCH 5/7] the script/functional return values should be handled by header arguments also, an echo header (like in R) argument would probably be appropriate as well. --- org-babel.org | 67 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/org-babel.org b/org-babel.org index 414016fe6..c660aded9 100644 --- a/org-babel.org +++ b/org-babel.org @@ -413,34 +413,33 @@ Thanks Dan, These comments are invaluable. What do you think about this as a new list of priorities/requirements for the execution of source-code blocks. -1) Sessions - 1) we want the evaluation of the source code block to take place in a - session which can persist state (variables, current directory, - etc...). - 2) source code blocks can specify their session with a header argument - 3) each session should correspond to an Emacs comint buffer so that the - user can drop into the session and experiment with live code - evaluation. -2) Results - 1) each source-code block generates some form of results which (as - we have already implemented) is transfered into emacs-lisp - after which it can be inserted into the org-mode buffer, or - used by other source-code blocks - 2) when the results are translated into emacs-lisp, forced to be - interpreted as a scalar (dumping their raw values into the - org-mode buffer), as a vector (which is often desirable with R - code blocks), or interpreted on the fly (the default option). - Note that this is very nearly currently implemented through the - [[* DONE results-type header (vector/file)][results-type-header]]. - 3) there should be *two* means of collecting results from the - execution of a source code block. *Either* the value of the - last statement of the source code block, or the collection of - all that has been passed to STDOUT during the evaluation. +- Sessions + 1) we want the evaluation of the source code block to take place in a + session which can persist state (variables, current directory, + etc...). + 2) source code blocks can specify their session with a header argument + 3) each session should correspond to an Emacs comint buffer so that the + user can drop into the session and experiment with live code + evaluation. +- Results + 1) each source-code block generates some form of results which (as + we have already implemented) is transfered into emacs-lisp + after which it can be inserted into the org-mode buffer, or + used by other source-code blocks + 2) when the results are translated into emacs-lisp, forced to be + interpreted as a scalar (dumping their raw values into the + org-mode buffer), as a vector (which is often desirable with R + code blocks), or interpreted on the fly (the default option). + Note that this is very nearly currently implemented through the + [[* DONE results-type header (vector/file)][results-type-header]]. + 3) there should be *two* means of collecting results from the + execution of a source code block. *Either* the value of the + last statement of the source code block, or the collection of + all that has been passed to STDOUT during the evaluation. -**** header argument or return line +**** header argument or return line (*header argument*) - - rather than using a header argument to specify how the return value + Rather than using a header argument to specify how the return value should be passed back, I'm leaning towards the use of a =#+RETURN= line inside the block. If such a line *is not present* then we default to using STDOUT to collect results, but if such a line *is @@ -450,6 +449,22 @@ for the execution of source-code blocks. of implementation and finding which statement is the last statement. + Having given this more thought, I think a header argument is + preferable. The =#+return:= line adds new complicating syntax for + something that does little more than we would accomplish through + the addition of a header argument. The only benefit being that we + know where the final statement starts, which is not an issue in + those languages which contain 'last value' operators. + + new header =:results= arguments + - script :: explicitly states that we want to use STDOUT to + initialize our results + - return_last :: stdout is ignored instead the *value* of the final + statement in the block is returned + - echo :: means echo the contents of the source-code block along + with the results (this implies the *script* =:results= + argument as well) + *** TODO rework evaluation lang-by-lang [0/4] This should include... From 304a59ef1ae46b5dcb00b5506050861d1b89ff99 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Thu, 11 Jun 2009 20:41:23 -0400 Subject: [PATCH 6/7] Initial design plan for off-the-shelf 'actions', like in org-R --- org-babel.org | 104 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/org-babel.org b/org-babel.org index c660aded9..d0db63fff 100644 --- a/org-babel.org +++ b/org-babel.org @@ -114,7 +114,7 @@ table, allowing the test suite to be run be evaluation of the table and the results to be collected in the same table. -* Tasks [22/38] +* Tasks [22/39] ** TODO Create objects in top level (global) environment [0/6] *sessions* @@ -630,19 +630,24 @@ $0 waiting for guidance from those more familiar with yasnippets -** TODO resolve references to other buffers +** TODO resolve references to other org buffers/files This would allow source blocks to call upon tables, source-blocks, - and results in other buffers. + and results in other org buffers/files. See... - [[file:lisp/org-babel-ref.el::TODO%20allow%20searching%20for%20names%20in%20other%20buffers][org-babel-ref.el:searching-in-other-buffers]] - [[file:lisp/org-babel.el::defun%20org-babel%20find%20named%20result%20name][org-babel.el#org-babel-find-named-result]] - +** TODO resolve references to other non-org files + - tabular data in .csv, .tsv etc format + - files of interpreted code: anything stopping us giving such files + similar status to a source code block? + - Would be nice to allow org and non-org files to be remote ** TODO figure out how to handle errors during evaluation R has a try function, with error handling, along the lines of python. I bet ruby does too. Maybe more of an issue for functional style; in my proposed scripting style the error just gets dumped to the org buffer and the user is thus alerted. + ** TODO figure out how to handle graphic output This is listed under [[* graphical output][graphical output]] in out objectives. @@ -653,6 +658,68 @@ results in the buffer. Then if there is a combination of =silent= and =file= =:results= headers we could drop the results to a temp buffer and pop open that buffer... +** TODO Finalise behaviour regarding vector/scalar output +*** DONE Stop spaces causing vector output +This simple example of multilingual chaining produces vector output if +there are spaces in the message and scalar otherwise. + +[Not any more] + +#+begin_src R :var msg=msg-from-python +paste(msg, "und R", sep=" ") +#+end_src + +#+resname: +: org-babel speaks elisp y python und R + +#+srcname: msg-from-python +#+begin_src python :var msg=msg-from-elisp +msg + " y python" +#+end_src + +#+srcname: msg-from-elisp +#+begin_src emacs-lisp :var msg="org-babel speaks" +(concat msg " elisp") +#+end_src + +** TODO re-implement helper functions from org-R +*** Initial statement [Eric] + Much of the power of org-R seems to be in it's helper functions for + the quick graphing of tables. Should we try to re-implement these + functions on top of org-babel? + + I'm thinking this may be useful both to add features to org-babel-R and + also to potentially suggest extensions of the framework. For example + one that comes to mind is the ability to treat a source-code block + like a function which accepts arguments and returns results. Actually + this can be it's own TODO (see [[* source blocks as functions][source blocks as functions]]). +*** Objectives + - We want to provide convenient off-the-shelf actions + (e.g. plotting data) that make use of our new code evaluation + environment but do not require any actual coding. +*** Design proposal + - *Input data* will be specified using the same mechanism as :var + references, thus the input data may come from a table, or + another source block, and it is initially available as an elisp + data structure. + - We introduce a new #+ line, e.g. #+BABELDO. C-c C-c on that + line will apply an *action* to the referenced data. + - *Actions correspond to source blocks*: our library of available + actions will be a library of org-babel source blocks. Thus the + code for executing an action, and the code for dealing with the + output of the action will be the same code as for executing + source blocks in general + - Optionally, the user can have the relevant source block inserted + into the org buffer after the (say) #+BABELDO line. This will + allow the user to fine tune the action by modifying the code + (especially useful for plots). + - So maybe a #+BABELDO line will have header args + - :data (a reference to a table or source code block) + - :action (or should that be :srcname?) which will be something + like :action pie-chart, referring to a source block which will + be executed with the :data referent passed in using a :var arg. + - :showcode or something controlling whether to show the code + ** TODO share org-babel how should we share org-babel? @@ -823,24 +890,6 @@ tabel Another example is in the [[*operations%20in%20on%20tables][grades example]]. -** PROPOSED Are we happy with current behaviour regarding vector/scalar output? -This simple example of multilingual chaining produces vector output if -there are spaces in the message and scalar otherwise. - -#+begin_src R :var msg=msg-from-python -paste(msg, "und_R", sep="_") -#+end_src - -#+srcname: msg-from-python -#+begin_src python :var msg=msg-from-elisp -msg + "_y_python" -#+end_src - -#+srcname: msg-from-elisp -#+begin_src emacs-lisp :var msg="org-babel_speaks" -(concat msg "_elisp") -#+end_src - ** PROPOSED conversion between org-babel and noweb (e.g. .Rnw) format I haven't thought about this properly. Just noting it down. What Sweave uses is called "R noweb" (.Rnw). @@ -884,17 +933,6 @@ This would allow for display of images upon export providing functionality similar to =org-exp-blocks= only in a more general manner. -** PROPOSED re-implement helper functions from org-R -Much of the power of org-R seems to be in it's helper functions for -the quick graphing of tables. Should we try to re-implement these -functions on top of org-babel? - -I'm thinking this may be useful both to add features to org-babel-R and -also to potentially suggest extensions of the framework. For example -one that comes to mind is the ability to treat a source-code block -like a function which accepts arguments and returns results. Actually -this can be it's own TODO (see [[* source blocks as functions][source blocks as functions]]). - ** DEFERRED use textConnection to pass tsv to R? When passing args from the org buffer to R, the following route is used: arg in buffer -> elisp -> tsv on file -> data frame in R. I From 9bf504e20795e20cc77d0ff994c1f3b37f11d85a Mon Sep 17 00:00:00 2001 From: Eric Schulte Date: Fri, 12 Jun 2009 13:35:46 -0700 Subject: [PATCH 7/7] brought in org-babel.org from session --- org-babel.org | 211 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 185 insertions(+), 26 deletions(-) diff --git a/org-babel.org b/org-babel.org index d0db63fff..550019971 100644 --- a/org-babel.org +++ b/org-babel.org @@ -1,6 +1,6 @@ -#+OPTIONS: H:3 num:nil toc:t #+TITLE: org-babel --- facilitating communication between programming languages and people -#+SEQ_TODO: TODO PROPOSED | DONE DEFERRED REJECTED +#+SEQ_TODO: TODO PROPOSED | DONE DEFERRED REJECTED +#+OPTIONS: H:3 num:nil toc:t #+STARTUP: oddeven hideblocks * Introduction @@ -114,8 +114,8 @@ table, allowing the test suite to be run be evaluation of the table and the results to be collected in the same table. -* Tasks [22/39] -** TODO Create objects in top level (global) environment [0/6] +* Tasks [22/38] +** TODO Create objects in top level (global) environment [2/5] *sessions* *** initial requirement statement [DED] @@ -465,39 +465,138 @@ for the execution of source-code blocks. with the results (this implies the *script* =:results= argument as well) -*** TODO rework evaluation lang-by-lang [0/4] +*** DONE rework evaluation lang-by-lang [4/4] This should include... -- STDOUT and functional results +- functional results working with the comint buffer +- results headers + - script :: return the output of STDOUT + - write a macro which runs the first redirection, executes the + body, then runs the second redirection + - last :: return the value of the last statement + - + - sessions in comint buffers -**** TODO R +**** DONE Ruby [4/4] +- [X] functional results working with comint +- [X] script results +- [X] ensure scalar/vector results args are taken into consideration +- [X] ensure callable by other source block -#+srcname: task-R-with-inf-process-buffer -#+begin_src R -a <- 8 -b <- 9 -c <- 10 +#+srcname: ruby-use-last-output +#+begin_src ruby :results replace +a = 2 +b = 4 +c = a + b +[a, b, c, 78] +#+end_src + +#+resname: ruby-use-last-output +| 2 | 4 | 6 | 78 | + +#+srcname: task-call-use-last-output +#+begin_src ruby :var last=ruby-use-last-output :results replace +last.flatten.size + 1 +#+end_src + +#+resname: task-call-use-last-output +: 5 + +***** ruby sessions + +#+srcname: first-ruby-session-task +#+begin_src ruby :session schulte :results silent +schulte = 27 +#+end_src + +#+srcname: second-ruby-session-task +#+begin_src ruby :session schulte :results silent +schulte + 3 +#+end_src + +#+srcname: without-the-right-session +#+begin_src ruby :results silent +schulte +#+end_src + +**** DONE R [4/4] + +- [X] functional results working with comint +- [X] script results +- [X] ensure scalar/vector results args are taken into consideration +- [X] ensure callable by other source block + +To redirect output to a file, you can use the =sink()= command. + +#+srcname: task_R_B +#+begin_src R :results value vector silent +a <- 9 +b <- 10 +b - a a + b #+end_src -**** TODO Ruby - -#+srcname: ruby-use-last-output -#+begin_src ruby -a = 1 -b = 2 -c = 3 -(a + b) * c +#+srcname: task-R-use-other-output +#+begin_src R :var twoentyseven=task_R_B() :results replace value +83 +twoentyseven + 9 #+end_src -**** TODO Python +#+resname: task-R-use-other-output +: 28 -**** TODO Shells +**** DONE Python [4/4] +- [X] functional results working with comint +- [X] script results +- [X] ensure scalar/vector results args are taken into consideration +- [X] ensure callable by other source block -*** TODO implement a *session* header argument +#+srcname: task-new-eval-for-python +#+begin_src python :results silent output scalar +8 +9 +10 +#+end_src -use this header argument to override the default *session* buffer +#+srcname: task-use-new-eval +#+begin_src python :var tasking=task-new-eval-for-python() :results replace +tasking + 2 +#+end_src + +#+resname: task-use-new-eval +: 12 + +**** DONE Shells [4/4] +- [X] functional results working with comint +- [X] script results +- [X] ensure scalar/vector results args are taken into consideration +- [X] ensure callable by other source block + +#+srcname: task-shell-new-evaluation +#+begin_src sh :results silent value scalar +echo 'eric' +date +#+end_src + +#+srcname: task-call-other-shell +#+begin_src sh :var other=task-shell-new-evaluation() :results replace scalar +echo $other ' is the old date' +#+end_src + +#+resname: task-call-other-shell +: $ Fri Jun 12 13:08:37 PDT 2009 is the old date + +*** TODO implement a *session* header argument [0/4] +=:session= header argument to override the default *session* buffer + +**** TODO R + + + +**** TODO ruby +**** TODO python +**** TODO shell *** TODO function to bring up inferior-process buffer @@ -510,6 +609,37 @@ up the inf-proc buffer using =pop-to-buffer=. Callable with a prefix argument to specify how many lines should be dumped into the source-code buffer. +*** REJECTED comint notes + +Implementing comint integration in [[file:lisp/org-babel-comint.el][org-babel-comint.el]]. + +Need to have... +- handling of outputs + - split raw output from process by prompts + - a ring of the outputs, buffer-local, `org-babel-comint-output-ring' + - a switch for dumping all outputs to a buffer +- inputting commands + +Lets drop all this language specific stuff, and just use +org-babel-comint to split up our outputs, and return either the last +value of an execution or the combination of values from the +executions. + +**** comint filter functions +: ;; comint-input-filter-functions hook process-in-a-buffer +: ;; comint-output-filter-functions hook function modes. +: ;; comint-preoutput-filter-functions hook +: ;; comint-input-filter function ... + +#+srcname: obc-filter-ruby +#+begin_src ruby :results last +1 +2 +3 +4 +5 +#+end_src + ** TODO support for working with =*Org Edit Src Example*= buffers [1/4] *** TODO set buffer-local-process variables appropriately [DED] I think something like this would be great. You've probably @@ -647,7 +777,6 @@ waiting for guidance from those more familiar with yasnippets python. I bet ruby does too. Maybe more of an issue for functional style; in my proposed scripting style the error just gets dumped to the org buffer and the user is thus alerted. - ** TODO figure out how to handle graphic output This is listed under [[* graphical output][graphical output]] in out objectives. @@ -890,6 +1019,24 @@ tabel Another example is in the [[*operations%20in%20on%20tables][grades example]]. +** PROPOSED Are we happy with current behaviour regarding vector/scalar output? +This simple example of multilingual chaining produces vector output if +there are spaces in the message and scalar otherwise. + +#+begin_src R :var msg=msg-from-python +paste(msg, "und_R", sep="_") +#+end_src + +#+srcname: msg-from-python +#+begin_src python :var msg=msg-from-elisp +msg + "_y_python" +#+end_src + +#+srcname: msg-from-elisp +#+begin_src emacs-lisp :var msg="org-babel_speaks" +(concat msg "_elisp") +#+end_src + ** PROPOSED conversion between org-babel and noweb (e.g. .Rnw) format I haven't thought about this properly. Just noting it down. What Sweave uses is called "R noweb" (.Rnw). @@ -1471,7 +1618,19 @@ This could probably be added to [[file:lisp/org-babel-script.el][org-babel-scrip (see [[* file result types][file result types]]) -* Bugs [11/14] +* Bugs [11/15] + +** TODO when reading results from =#+resname= line + +Errors when trying to read from resname lines. + +#+resname: bug-in-resname +: 8 + +#+srcname: bug-in-resname-reader +#+begin_src emacs-lisp :var buggy=bug-in-resname() :results silent +buggy +#+end_src ** TODO non-orgtbl formatted lists for example