#!/bin/sh # This is a shell archive (produced by GNU sharutils 4.2). # To extract the files from this archive, save it to some FILE, remove # everything before the `!/bin/sh' line above, then type `sh FILE'. # # Made on 1998-10-02 17:55 EDT by . # Source directory was `/home/jmc/Sbook'. # # Existing files will *not* be overwritten unless `-c' is specified. # # This shar contains: # length mode name # ------ ---------- ------------------------------------------ # 3489 -rw-rw-r-- commitAssigns.html # 2325 -rw-rw-r-- databaseClass.html # 2175 -rw-rw-r-- errata.html # 3594 -rw-rw-r-- index.html # 3505 -rw-rw-r-- lazyEval.html # 4232 -rw-rw-r-- methodDispatch.html # 3726 -rw-rw-r-- oldUseMethod.html # 2001 -rw-rw-r-- selectMethod.html # 7008 -rw-rw-r-- DatabaseClasses/dumpDataBaseClass.html # 9697 -rw-rw-r-- DatabaseClasses/index.html # 7748 -rw-rw-r-- DatabaseClasses/symbDBClass.html # 3031 -rw-rw-r-- DatabaseClasses/ttt.html # 1040 -rw-rw-r-- Documentation/FutureProjects.html # 2789 -rw-rw-r-- Documentation/Overview.html # 8126 -rw-rw-r-- Documentation/Programming.html # 460 -rw-rw-r-- Documentation/Rendering.html # 2911 -rw-rw-r-- Documentation/WritingDocumentation.html # 2003 -rw-rw-r-- Documentation/index.html # 2080 -rw-rw-r-- Documentation/links.html # save_IFS="${IFS}" IFS="${IFS}:" gettext_dir=FAILED locale_dir=FAILED first_param="$1" for dir in $PATH do if test "$gettext_dir" = FAILED && test -f $dir/gettext \ && ($dir/gettext --version >/dev/null 2>&1) then set `$dir/gettext --version 2>&1` if test "$3" = GNU then gettext_dir=$dir fi fi if test "$locale_dir" = FAILED && test -f $dir/shar \ && ($dir/shar --print-text-domain-dir >/dev/null 2>&1) then locale_dir=`$dir/shar --print-text-domain-dir` fi done IFS="$save_IFS" if test "$locale_dir" = FAILED || test "$gettext_dir" = FAILED then echo=echo else TEXTDOMAINDIR=$locale_dir export TEXTDOMAINDIR TEXTDOMAIN=sharutils export TEXTDOMAIN echo="$gettext_dir/gettext -s" fi touch -am 1231235999 $$.touch >/dev/null 2>&1 if test ! -f 1231235999 && test -f $$.touch; then shar_touch=touch else shar_touch=: echo $echo 'WARNING: not restoring timestamps. Consider getting and' $echo "installing GNU \`touch', distributed in GNU File Utilities..." echo fi rm -f 1231235999 $$.touch # if mkdir _sh02895; then $echo 'x -' 'creating lock directory' else $echo 'failed to create lock directory' exit 1 fi # ============= commitAssigns.html ============== if test -f 'commitAssigns.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'commitAssigns.html' '(file already exists)' else $echo 'x -' extracting 'commitAssigns.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'commitAssigns.html' && Commiting S Assignments X

Commiting S Assignments

X This is an extension of the discussion (e.g., on page 178) of assignment expressions, particularly at the top level. The model for evaluating these expressions is simple, but it can have some unexpected consequences. In addition, we discuss the related options(immediate="true") and its effect.

The assignment operation

    scaleRes = myscale(curfit$residuals)
always has the effect of associating the dataset name, "scaleRes" with the evaluated expression, in the local frame–the frame in which the expression is evaluated. When that frame is frame 1, the task frame, the evaluator takes an additional action, depending on the value of the "immediate" option. For the moment, assume that option has its default value, FALSE. X In this case the evaluator in effect schedules an assignment operation to be added to the exit actions for frame 1.
    if(error.level() == 0)
X      assign("scaleRes", scaleRes, where = 1)
The actual computation is done internally in the evaluator, but the call to assign is the way to think about what happens. The call will find the value of scaleRes on frame 1 and copy it to the working database. The condition that the error level be zero protects commitment of the assignment in the case that the task ends in an error.

So what could be confusing here? Well, mainly that the commitment is an exit action at the end of the task. It's the value of the object in frame 1 at that time that gets copied. Intermediate assignments or replacements to the object in frame 1 will alter the contents of the object copied. This is obvious with other assign operations, but it also applies to calls to the assign function on frame 1:

    assign("scaleRes", newScaleRes, frame=1)
If this takes place after the first assignment, this will be the object saved. Conversely, assign calls directly to the working database during the task will be overwritten, naturally, by the corresponding call on exit:
    assign("scaleRes", newScaleRes, where=1)
You almost certainly didn't want to do this!

One related point: removing the object from frame 1 will cause the evaluator to in effect cancel the assign call on exit.

Now, about the "immediate" option. When this is set to TRUE, the evaluator commits assignments to the working data immediately; in effect, the assign call is done right after assigning the object to frame 1. You lose the protection of the commitment, but that may be just what you want, to be sure in case of problems that you have the latest version of the object on the working data. The function synchronize does a related task: it commits the assignments when it's called, so you can have commitment on demand rather than immediately or at the end of the task. X


John Chambers<jmc@research.bell-labs.com>
Last modified: Fri Oct 2 17:54:48 EDT 1998 SHAR_EOF $shar_touch -am 1002175498 'commitAssigns.html' && chmod 0664 'commitAssigns.html' || $echo 'restore of' 'commitAssigns.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'commitAssigns.html:' 'MD5 check failed' a7460f08002c052acdd3fd4d52b778e6 commitAssigns.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'commitAssigns.html'`" test 3489 -eq "$shar_count" || $echo 'commitAssigns.html:' 'original size' '3489,' 'current size' "$shar_count!" fi fi # ============= databaseClass.html ============== if test -f 'databaseClass.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'databaseClass.html' '(file already exists)' else $echo 'x -' extracting 'databaseClass.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'databaseClass.html' && Database Classes X

Database Classes

Section 5.3 of Programming with Data mentions a number of different ways to define an S database, including user-defined database classes. Not much more is said about these in the book.

Here they are. The idea quite powerful; although a moderate amount of programming is needed to add a new kind of database class, the results can usually be reused for other, similar applications. This page outlines what is needed, and gives an example using the S symbolic dump format. The example has some mild practicality, in providing a portable database that could be used in a network file system over diverse hardware. It's main virtue, though, is to work out the essential ideas in a context that doesn't require discussing any other software, such as database management systems.

A database class is any class that extends the virtual class database. Attaching an object from this class causes S to look for corresponding methods for a set of basic database functions, to read, write, remove objects. Once the required set of such methods is defined, you can attach an object of this class and use the resulting database like any other.

Notice that this is different from attaching a list-like object as a database. In that case, all the data resided in the single object. For database classes, on the other hand, the database object is usually just some identifier for a directory, database, URL or whatever. It's just there to tell the evaluator to use corresponding methods to deal with the data.

Let's do an example. In this case we are going to use a directory in the file system as our database. But instead of relying on the standard S binary representation for objects, we will read and write files using the text-only "symbolic dump format" (pp 237-241). The class just needs to extend database and contain the path for the directory.



John Chambers<jmc@research.bell-labs.com>
Last modified: Wed Aug 5 17:21:10 EDT 1998 SHAR_EOF $shar_touch -am 0805172198 'databaseClass.html' && chmod 0664 'databaseClass.html' || $echo 'restore of' 'databaseClass.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'databaseClass.html:' 'MD5 check failed' 10abf44eba62e7bc41bb8e914e8bc529 databaseClass.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'databaseClass.html'`" test 2325 -eq "$shar_count" || $echo 'databaseClass.html:' 'original size' '2325,' 'current size' "$shar_count!" fi fi # ============= errata.html ============== if test -f 'errata.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'errata.html' '(file already exists)' else $echo 'x -' extracting 'errata.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'errata.html' && Programming with Data: Errata

Programming with Data: Errata

X

Page 103, Table 3.10, the first entry should all be on one line, and there is an extra "." at the end of the caption.

Page 111, Table 3.17, "traceMethod'` should be one column to the right.

Page 134, line 3, missing final ")" in the call to eval().

Page 185, line 17, should read 'name "..1" refers to ...'.

Page 196, last paragraph should read "Section 7.5 deals with ...".

Page 198, line -1, should read "where e is of class expression"

Page 212, third sentence of section 5.2 is unclear, better would be: "Each database is essentially a lookup table of non-empty strings, each string associated with an S object."

Page 215, line -3 of section 5.2.1 should read "... same semantics as referring...".

Page 216, line 11 should read "when an attach or detach call".

Page 217, line 3 in section 5.2.3, second sentence should read "All have names beginning with database. and completed ...".

Pages 245-7, the file name "rescale.S" should be "reScale.S", and the function rescale should be reScale in several occurrences on these pages.

Page 286, second paragraph, second sentence should read "Direct inclusion is restricted to unambiguous cases."

Page 340, second paragraph, the formal arguments to function "[" include a third argument, drop, so that the function declaration in the example should be

X      function(x, ..., drop = T)  track(x@x[...], x@y[...]))

Page 375, third line of display near the bottom of the page should be "writeLines", not "readLines".


Thanks to Tim Hesterberg, David James, and Brian Ripley. X X X
John Chambers<jmc@research.bell-labs.com>
Last modified: Tue Sep 8 11:29:12 EDT 1998 SHAR_EOF $shar_touch -am 0908112998 'errata.html' && chmod 0664 'errata.html' || $echo 'restore of' 'errata.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'errata.html:' 'MD5 check failed' d4231fc416ec61aba51e9965ea5ef1fb errata.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'errata.html'`" test 2175 -eq "$shar_count" || $echo 'errata.html:' 'original size' '2175,' 'current size' "$shar_count!" fi fi # ============= index.html ============== if test -f 'index.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'index.html' '(file already exists)' else $echo 'x -' extracting 'index.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'index.html' && X X Programming with Data

X Programming with Data
A Guide to the S Language

This book is a guide and reference to the version of S underlying S-Plus Versions 5.0 and later (not useful if you are programming with earlier versions of S-Plus). The book is published by Springer-Verlag. The ISBN is 0-387-98503-4 and the publication date 1998.

If you don't have the book and are interested, you can look at the preface and contents in postscript. X An earlier article X from the 1996 Interface Conference gives some underlying design X goals for the revisions in S. It is a little out of date (especially in not discussing the use of SGML as a model for the online documentation), but covers the general philosophy of the language.

This web site provides a focus for extensions, details and errata. The material either surfaced after the book was published or else carries the discussion beyond the scope of the book. There are also pointers to other topics related to S and the book, but not directly part of the language.


Detailed Discussions

The following pages extend the discussion of topics in the book to newer material and/or to details that just didn't make the cutoff.
X
 o X Method Dispatch X
Variations on choosing and invoking a method. X
 o X Database Classes X
make most anything into an S database. X
 o X Committing Assignments X
Details on how it works, and on options("immediate"). X
 o X Objects in an Old-Style Method X
If the old-style generic modifies objects before calling X UseMethod, what happens? X
 o X Lazy Evaluation and Missing Arguments X
Some details on when arguments are really evaluated, X and what happens if they're missing.

Documentation

X The book describes the essentials of S programming to use and produce online documentation for functions, classes and methods. The S documentation objects map into the SGML language, opening up many opportunities for working with the documentation using, for example, XML tools.
X
 o X The S Documentation System X
 o X Overview X
 o X SGML tags X for S

X


Known Errors in the Book X


X SHAR_EOF $shar_touch -am 1002175298 'index.html' && chmod 0664 'index.html' || $echo 'restore of' 'index.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'index.html:' 'MD5 check failed' 7006aaa8070e0c72d947fe29eee9a4e2 index.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'index.html'`" test 3594 -eq "$shar_count" || $echo 'index.html:' 'original size' '3594,' 'current size' "$shar_count!" fi fi # ============= lazyEval.html ============== if test -f 'lazyEval.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'lazyEval.html' '(file already exists)' else $echo 'x -' extracting 'lazyEval.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'lazyEval.html' && Lazy Evaluation and Missing Arguments X

Lazy Evaluation and Missing Arguments

X

The S evaluation model for function calls is unusual in that arguments are evaluated during evaluation of the body of the called function, not before the call, and then only when the value is really needed. Argument evaluation is described on pages 181-183 of "Programming with Data"; in particular, page 182 describes when the actual evaluation takes place; roughly, when using an interface to C, etc., when copying as for a local assignment, and when looking for a method. For the first two cases, a missing with no default produces an error then, but not before. For method search, class "missing" is legitimate in a signature, and not an error.

It might not be obvious, to say the least, what are all the consequences of lazy evaluation. There are some examples in the book that illustrate useful cases of allowing missing arguments. Also, other examples can arise in which there are two possible expressions, supplied as two arguments, but with the understanding that the called function will choose only one of these to evaluate.

As yet another variant, consider the following fragment. It's useless as is, but meant to illustrate another possible technique with lazy evaluation. Function b passes its argument to function a, but doesn't want to call a with no arguments if its own argument is missing. Instead it computes a flag to warn a not to use the argument.

> b = function(x) a(x, missing(x))
> a = function(y, useDefault) if(missing(y) || useDefault) 0 else x
> b(1)
[1] 1
> b()
[1] 0
Obviously, this is trivial as it stands, but embedded into a more elaborate context it might be a convenient simplification of the design of b.

One natural confusion might be to think that missing(y) is TRUE in the body of a. But this is not what's happening: the argument to a is never missing in the call from b. What the object passed to a actually is, on the other hand, we should never know or care.

A subtle implementation detail is that the S evaluator has to handle such situations by checking the actual object reference (to use Java terminology) rather than the contents of the object. Generally, one can't say in a language like S that any actual object is "not an object"; you get into some impossible circularity when you try to apply that idea in general.

In the example above, doing anything much other than missing(y) to the argument is going to generate an error (effectively preventing you from peeking at the actual object passed down). The reason is not a problem in a, but rather that evaluating the argument there requires evaluating the argument in b, which is missing with no default. Hence the error message below.

> b = function(x)a(x)
> a = function(y)class(y)
> b(pi)
[1] "numeric"
> b()
Problem in b(): argument "x" is a missing with no default 
X
John Chambers<jmc@research.bell-labs.com>
Last modified: Wed Jul 29 11:25:18 EDT 1998 SHAR_EOF $shar_touch -am 0729112598 'lazyEval.html' && chmod 0664 'lazyEval.html' || $echo 'restore of' 'lazyEval.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'lazyEval.html:' 'MD5 check failed' b3ed72c8f872923aecfd1742d57fd456 lazyEval.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'lazyEval.html'`" test 3505 -eq "$shar_count" || $echo 'lazyEval.html:' 'original size' '3505,' 'current size' "$shar_count!" fi fi # ============= methodDispatch.html ============== if test -f 'methodDispatch.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'methodDispatch.html' '(file already exists)' else $echo 'x -' extracting 'methodDispatch.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'methodDispatch.html' && Method Dispatch

Variations on Dispatching a Method

X

Standard S Method Dispatch


The standard generic function in S has as its body, naturally,
standardGeneric("foo")

which calls the internal C code to select a method based on the classes of the arguments. Section 4.9 discusses how this works; roughly, for each formal argument involved in the method selection, the evaluator matches the actual argument classes (or "missing" if the argument is missing) against the stored table of methods. The closest matching method is selected (maybe the default method) and each argument, coerced to the corresponding class, is assigned in the evaluation frame for the function. Finally, the body of the selected method is evaluated in this frame, and the result of this evaluation becomes the value of the call.

By definition, coercing the classes means that the actual argument, when the method is evaluated, has the class specified in the chosen method's signature, not necessarily its original class. This is essential in an object-based system, because the writer of the method needs to know that it's dealing with a known class (and extensions can involve coerce methods, so the actual argument may not look anything like the selected class). The partial exception is that virtual classes in the signature always expect the actual object to have some non-virtual class.

If you've used other object-oriented languages such as Java or C++, this may seem odd or restrictive, but it is part of the essential evaluation model for S. You don't need to be restricted by it, however, if you have an example where it really doesn't fit what you want to do. The S evaluator is available directly, so you can program a different method dispatch mechanism. These notes describe how.

Virtual Method Dispatch


In this mechanism, the method is selected but the arguments are not coerced. Instead the body is evaluated directly. It will see the actual arguments just as they are supplied by the calling function.
foo = function(x, y) {
X    eval(selectMethod())
}
(This relies on an
extension to the selectMethod function post-dating the book, in which it uses the actual arguments to define the method selection.)

The methods for virtual dispatch must be specially designed. As mentioned before, they can't in general assume anything about the actual form (slots, for example) in the arguments. They can assume that the arguments extend the class for which the method was written. I imagine most such methods would extract some general information about the object (its class, for example), and then call the as function to coerce the argument to a known form. X

Dispatch Without Inheritance

In this version, the method writer is guaranteed that the actual argument has the specified class, not merely that it extends that class. This is the appropriate form for functions that need to believe they have all the information they could need about the object (dput is an example). In this case the generic function has a form such as
dput = function(x = NULL, file = stdout())
{
X  sig = selectMethodSignature("dput", class(x))
X  if(identical(sig, class(x)))
X    standardGeneric("dput")
X  else eval(functionBody(selectMethod("dput", character())))
X  invisible(x)
}

If the selected method matches the actual classes, do the standard thing; otherwise, select and evaluate the default method. If the function can have special methods for missing arguments, the body of the generic function will have to use something a little fancier than class(x), say
if(missing(x)) "missing" else class(x)

X X
John Chambers<jmc@research.bell-labs.com>
Last modified: Sat May 2 16:52:36 EDT 1998 SHAR_EOF $shar_touch -am 0502165298 'methodDispatch.html' && chmod 0664 'methodDispatch.html' || $echo 'restore of' 'methodDispatch.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'methodDispatch.html:' 'MD5 check failed' cab85cc2a9595f8ee02913660db99c61 methodDispatch.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'methodDispatch.html'`" test 4232 -eq "$shar_count" || $echo 'methodDispatch.html:' 'original size' '4232,' 'current size' "$shar_count!" fi fi # ============= oldUseMethod.html ============== if test -f 'oldUseMethod.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'oldUseMethod.html' '(file already exists)' else $echo 'x -' extracting 'oldUseMethod.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'oldUseMethod.html' && Detailed Behavior of UseMethod X

Detailed Behavior of UseMethod

The old-style methods are initially invoked by a call to UseMethod. What happens next is described in section A.6 of Statistical Methods in S. The description there is actually a ambiguous in the case that the generic function modifies the first argument before calling UseMethod. The statement on page 467 that the method acts as if the call to the generic had been a call to the method is then in conflict with the statement on the next page that the first argument will have been evaluated. What if it's not only evaluated but then modified? On the whole, generic functions probably shouldn't do this very often--the philosophy of generic functions is that they let the methods do the computation. I don't believe anyone brought this inconsistency to our attention for six years or so, until Rob Gentleman pointed it out.

Implementing this model for method dispatch requires stepping outside the standard function-call model. In both the original and the current implementation the evaluator uses some special techniques, but not quite the same technique. As a result, the exact semantics vary a little. The current semantics seem slightly more consistent, and in fact stick somewhat more closely to the standard model. One sentence, on page 467 in Statistical Methods in S, is not literally true: the frame for the call to the generic does not become the frame for the method, but instead is replaced by a new frame, which has the same parent frame and the same actual arguments (rematched if the method has a different formal argument list) as the call to the generic. The call to the generic remains open until the method returns, when the frames for both calls will be popped. The values of objects in the frame reflect any re-assignments done before the call to UseMethod.

The important part of the semantics remains true in both implementations: the parent frame of the method is the frame of the caller to the generic function. So use of functions such as substitute in the method works naturally. As another aside, the comments on page 447 of Programming with Data are probably too pessimistic in this respect. Contrary to the assertion in point 1 on that page, most calls to substitute and sys.parent should behave as they did in the old days.

The following example illustrates the points made so far. It's to be hoped, though, that very few actual computations will end up depending on this level of detail.

> oldF
function(x, y, ...)
{ x = x + 1;  y = y + 1
X  UseMethod("oldF")
}
> oldF.foo
function(x, y, z)
{
X  sum = x + y + z
X  label = substitute(x)
X  sum
}
> trace(oldF.foo,exit=browser)
> Tx
[1] 1
attr(, "class"):
[1] "foo"
> oldF(Tx + 1, 2, 100 + 1)
On exit: Called from: oldF.foo(Tx + 1, 2, 100 + 1) 
b(oldF.foo)> x
[1] 3
attr(, "class"):
[1] "foo"
b(oldF.foo)> y
[1] 3
b(oldF.foo)> label
Tx + 1
b(oldF.foo)> q
[1] 107
attr(, "class"):
[1] "foo"
> oldClass(Tx)
[1] "foo"
In S-Plus 3.4, with the same example, x has the modified value, but y does not. I believe the reason is that all the arguments are re-matched and then x only (since it's known to have been evaluated) is copied into the new frame. X X
John Chambers<jmc@research.bell-labs.com>
Last modified: Thu Jun 25 15:06:46 EDT 1998 SHAR_EOF $shar_touch -am 0625150698 'oldUseMethod.html' && chmod 0664 'oldUseMethod.html' || $echo 'restore of' 'oldUseMethod.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'oldUseMethod.html:' 'MD5 check failed' e09cde755655ada3c39534eb5f44597a oldUseMethod.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'oldUseMethod.html'`" test 3726 -eq "$shar_count" || $echo 'oldUseMethod.html:' 'original size' '3726,' 'current size' "$shar_count!" fi fi # ============= selectMethod.html ============== if test -f 'selectMethod.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'selectMethod.html' '(file already exists)' else $echo 'x -' extracting 'selectMethod.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'selectMethod.html' && X X selectMethod Extensions X

Extensions to selectMethod and selectMethodSignature

X The calls to selectMethod and selectMethodSignature described in Programming with Data take as arguments the name of a generic function and a signature, that is, a vector of classes corresponding to some of the formal arguments to the function of that name. The functions return respectively a function definition or a signature corresponding to the method the evaluator would select for this function and signature.

For two reasons, one would like also to find the method or the signature corresponding to the current, actual call to the function. First, this is a useful prelude to doing some non-standard generic computations, such as virtual method dispatch. Second, the correct method cannot always be determined by the classes of the argument alone. Suppose, for example, the actual argument x has class "matrix", but the only methods defined correspond to "character" and "numeric". Then the method to be chosen could be either of these or the default, depending on the class of the data contained in the matrix.

In the extension, selectMethod and selectMethodSignature are called with no arguments. They then call internal code in the evaluator to carry out method selection for the actual arguments to the call from which selectMethod or selectMethodSignature was called. X X X


John Chambers<jmc@research.bell-labs.com>
Last modified: Wed Jun 24 15:13:42 EDT 1998 SHAR_EOF $shar_touch -am 0624151398 'selectMethod.html' && chmod 0664 'selectMethod.html' || $echo 'restore of' 'selectMethod.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'selectMethod.html:' 'MD5 check failed' 7fec2bcf73c0e9a22fd0e9c49775da4c selectMethod.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'selectMethod.html'`" test 2001 -eq "$shar_count" || $echo 'selectMethod.html:' 'original size' '2001,' 'current size' "$shar_count!" fi fi # ============= DatabaseClasses/dumpDataBaseClass.html ============== if test ! -d 'DatabaseClasses'; then $echo 'x -' 'creating directory' 'DatabaseClasses' mkdir 'DatabaseClasses' fi if test -f 'DatabaseClasses/dumpDataBaseClass.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'DatabaseClasses/dumpDataBaseClass.html' '(file already exists)' else $echo 'x -' extracting 'DatabaseClasses/dumpDataBaseClass.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'DatabaseClasses/dumpDataBaseClass.html' && Example Source for User-Defined Database Class X

Example Source for User-Defined Database Class

X Here is about the simplest definition possible for a user-defined class and the corresponding methods. The class writes and reads objects in symbolic dump format, in a directory specified when the object from the class is created. For compatibility with the S function data.dump, and to decrease the chance of errors from trying to read arbitrary files as objects, the file name has ".Sdata" appended to the object name.

With this definition, every method is a one-line function definition, and the one utility function, to create the appropriate path name, is just a call to paste. Only the generator function manages to run a little longer, mainly because it tries to be friendly about creating the directory if it doesn't exist, and because it stores the full path of the directory. X X

Class Definition

X Defines the class to extend the virtual class "database" and to contain one slot, for the string path name of the corresponding directory. X
    setClass("dumpDataBase",
X        representation("database", path = "character"))
X

Generator Function for the Class

X Takes as argument a character string for the directory's path and, optionally, a flag to say whether to create the directory if it does not exist. X
"dumpDataBase" =
X  ## make an object of class `"dumpDataBase"' corresponding to the directory
X  ## `path'.  The flag `create' allows or suppresses creating the directory if it
X  ## does not exist (by default, the user is asked, if S is in interactive mode).
X  function(path,
X           create = identical(dialog(paste("Directory \"", path,
X             "\"for database does not exist; create?", sep=""), c("y", "n"), 2),1))
{
X  path = as(path, "character")
X  if(length(path)!=1) stop("needed a single character string for path")
X  if(!isDirectory(path)) {
X    if(create) {
X      shell(paste("mkdir", path), mustWork=T)
X      file = paste(path, "objects", sep="/")
X      cat("", file=file, sep="")
X    }
X    else
X      warning("Directory \"", path,
X              "\"for database does not exist; path validity not checked")
X  }
X  if(isDirectory(path))
X    ## get a fully-qualified path
X    path = shell(paste("cd", path, "; pwd"))
X  new("dumpDataBase", path = path)
}
The reason for generating the fully-qualified path is that, otherwise, the behavior of the database object would depend on the directory in which the S process is running, very likely not what the user wants. X

Method for dbobjects: Listing the Object Names

X This method returns a character vector of the current object names in the directory. It is normally only called when the database is being attached. From then on, the S evaluation manager maintains its own table of the contents of the database. As with ordinary directory databases, this means that multiple processes using the same database simultaneously are not automatically co-ordinated. The method is just a one-line shell command, to get all the file names ending in ".Sdata", since we decided to write and read files with that suffix. A small free extension comes from using the find shell command: objects can actually be in subdirectories of the current directory. Their names will have an embedded "/" and the simple-minded methods shown here don't make subdirectories automatically, so the feature is probably worth about what it costs.
setMethod("dbobjects", "dumpDataBase",
function(database)
shell(paste("find", database@path, "-name '*.Sdata' -print| sed -e 's/.Sdata$//'"))
)
X

Method for dbread: Read an Object

X Given the database and the name of an object guaranteed to be in the datbase, this method reads it in. Since all objects are written in the standard S symbolic dump format by dataPut, this method just calls dataGet to do the reading. X
setMethod("dbread", "dumpDataBase", 
function(database, name)
dataGet(dumpDBPath(database, name))
)
X

Method for dbwrite: Write an Object

X This method takes the database, an object to write out, and the name the object should have when assigned in the database, by calling the dataPut function with the path for the given name.
setMethod("dbwrite", "dumpDataBase", 
function(database, name, object)
dataPut(object, dumpDBPath(database, name))
)
X

Method for dbremove: Remove an Object

X This function carries out the removal of the object, just by removing the corresponding file. Notice that it doesn't worry about whether the file exists: in principle, this method only gets called after the S evaluator determines that the object exists in the database's directory.
setMethod("dbremove", "dumpDataBase", 
function(database, name)
shell(paste("rm '", dumpDBPath(database, name), "'", sep=""), output=F, mustWork=T))
X

Method for dbexists: Does an Object Exist?

X This method is only called when there is no internal directory for the database in the S evaluator manager; that is, when the dbobjects method returns NULL; see the discussion of implementation without a directory. X

The Path Utility

X Here is the function to create the path string for an object name:
dumpDBPath = function(database, name)
X  paste(database@path, "/", name, ".Sdata", sep="")
X

Implementation without Directory

X The following class definition extends the previous class to allow individual databases to be attached without creating an internal directory in the S evaluation manager.
setClass("dumpFilesDB", representation("dumpDataBase", directory = "logical"))
X X The only method that needs changing is dbobjects, which now returns NULL if it finds that the directory slot is FALSE. setMethod("dbobjects", "dumpFilesDB", function(database) if(database@directory) shell(paste("find", database@path, X "-name '*.Sdata' -print| sed -e 's/.Sdata$//'")) else NULL )
X X
John Chambers<jmc@research.bell-labs.com>
Last modified: Fri Oct 2 14:05:50 EDT 1998 SHAR_EOF $shar_touch -am 1002140598 'DatabaseClasses/dumpDataBaseClass.html' && chmod 0664 'DatabaseClasses/dumpDataBaseClass.html' || $echo 'restore of' 'DatabaseClasses/dumpDataBaseClass.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'DatabaseClasses/dumpDataBaseClass.html:' 'MD5 check failed' 8c646dd06dc19eebeca2405e61e238e9 DatabaseClasses/dumpDataBaseClass.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'DatabaseClasses/dumpDataBaseClass.html'`" test 7008 -eq "$shar_count" || $echo 'DatabaseClasses/dumpDataBaseClass.html:' 'original size' '7008,' 'current size' "$shar_count!" fi fi # ============= DatabaseClasses/index.html ============== if test -f 'DatabaseClasses/index.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'DatabaseClasses/index.html' '(file already exists)' else $echo 'x -' extracting 'DatabaseClasses/index.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'DatabaseClasses/index.html' && User-Defined Database Classes X

Database Classes

Section 5.3 of Programming with Data mentions a number of different ways to define an S database, including user-defined database classes. Not much more is said about these in the book.

Here they are. The idea is quite powerful. The S evaluation manager treats user-defined databases essentially just like other types of databases, until it reaches the point at which it needs to take an explicit action: reading, writing, or removing an object, for example. At this point it calls a corresponding generic function: dbread, dbwrite, and dbremove in these cases. The designer of the database class provides methods for these functions, all of which get the database object as their first argument. An important part of the philosophy is that the evaluation manager is responsible for determining in most cases that we really need to read, write, or remove the corresponding object. The database methods do not worry about issues such as committing assignments or whether there are multiple versions of the same named object on other databases. X Although a moderate amount of programming is needed to add a new kind of database class, the results can usually be reused for other, similar applications. This page outlines what is needed, and gives an example using the S symbolic dump format. The example has some mild practicality, in providing a portable database that could be used in a network file system over diverse hardware. Its main virtue, though, is to work out the essential ideas in a context that doesn't require discussing any other software, such as database management systems.

A database class is any class that extends the virtual class database. Once the required methods for dbread, etc. are defined or inherited, you can attach an object of the class and use the resulting database like any other.

Notice that this is different from attaching a list-like object as a database. In that case, all the data resided in the single object. For database classes, on the other hand, the database object usually identifies a directory, database, URL or other external resource that will be used to store and retrieve data. The database object's class has the key role of telling the evaluator to use corresponding methods to deal with the low-level access to the database.

An Example

Let's do a very simple example. In this case we are going to use a directory in the file system as our database. But instead of relying on the standard S binary representation for objects, we will read and write files using the text-only "symbolic dump format" (pp 237-241). As for all user-defined database classes, the new class extends the virtual class "database". Other than that, the representation only contains the path for the directory.
    setClass("dumpDataBase", representation(path = "character"))
X

We will store objects in separate files in the directory; as a heuristic we'll append ".Sdata" to the object name to get the file name. This is consistent with the data.dump function, and also helps avoid confusion if the user creates some other files in the same directory.

As with most class definitions, the next step is to write a generating function for objects from the class. The example generator function takes the path name of a directory as argument. If the directory doesn't exist, the function optionally creates it. If the directory exists or is created, the path stored in the object is a global, not a local path, so that future use of the database object is unambiguous.

The next step is to write some methods for the database operations. The four critical generic functions and the corresponding methods are:

All the examples in this case are trivial one-line methods, but looking at them will help you understand what the methods can assume and what they need to do. In addition to these four methods, several others may optionally be defined.

That's it, apart from a one-line utility that returns the path corresponding to an object name. Even this simplistic example has some utility. Notice that the contents of the database are now portable; we could attach and use such a database on a networked file system, regardless of the computer on which S was running.

This is a very compact implementation, but lacking in generality. One problem is that only certain object names are valid file names. A somewhat fancier alternative definition of the database class allows arbitrary object names at the cost of keeping a local table to map object names into file names. X X

Databases without Directories

X Normally, the S evaluator manager maintains an internal table of the names of all the objects currently on a database. As S tasks alter the database, this directory is updated to reflect assignments and removals of objects. For user-defined database classes, the internal directory is initialized when the database is attached, via a call from the evaluator manager to the function dbobjects. This should return a vector of the current object names. There is only this one call each time the database is attached. From then on, the evaluator believes in its own internal table.

Occasionally, you may not want S to have such an internal table. In some applications, it may be expensive or even impossible to define all the objects in the database. At other times, you may not want to count on the internal view of the database. For example, if other processes are expected to be updating the directory during the session, you will not be guaranteed a current version of the database: S will not know about objects that have been added or removed by other software. On the other hand, not having an internal directory is a serious problem if a database is to be used on the regular S search path. Efficiency suffers greatly if each attempt to search for an S function, for example, has to run a complex computation to test whether the corresponding objects exists on a particular database. There are dangers of getting trapped in infinite loops, if the function is needed to evaluate one of the database methods themselves.

The approach taken is to allow user-defined database classes without internal directories, but not on the search list. Only databases attached with purpose="data" are allowed to dispense with directories. These databases are accessed only when an explicit where= argument is supplied. See page 226 of Programming with Data for this use of attach.

You indicate that a database should be attached without a directory by having the dbobjects method return the NULL object. If databases from your class never should have directories, just make sure that the default dbobjects method is used; i.e., don't have a method for your class or for any class that your class extends.

If the database has no directory, then there must be a method for the function dbexists; reasonably so, since the evaluator and the user need to be able to test whether an object exists on the database, rather than trying to read the object and getting an error.

Let's extend our previous example to make directories optional. We'll call this class "dumpFilesDB"; it extends the previous class to add a flag for having a directory.

setClass("dumpFilesDB", representation("dumpDataBase", directory = "logical"))
The generator function is the same, with an added argument for whether to create a directory. All the methods are inherited from "dumpDataBase", except for dbobjects, which returns NULL if the directory flag in the object is FALSE. X X X X
John Chambers<jmc@research.bell-labs.com>
Last modified: Fri Oct 2 17:16:17 EDT 1998 SHAR_EOF $shar_touch -am 1002171698 'DatabaseClasses/index.html' && chmod 0664 'DatabaseClasses/index.html' || $echo 'restore of' 'DatabaseClasses/index.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'DatabaseClasses/index.html:' 'MD5 check failed' 5bb45339a7baed1830e22eeb5094be27 DatabaseClasses/index.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'DatabaseClasses/index.html'`" test 9697 -eq "$shar_count" || $echo 'DatabaseClasses/index.html:' 'original size' '9697,' 'current size' "$shar_count!" fi fi # ============= DatabaseClasses/symbDBClass.html ============== if test -f 'DatabaseClasses/symbDBClass.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'DatabaseClasses/symbDBClass.html' '(file already exists)' else $echo 'x -' extracting 'DatabaseClasses/symbDBClass.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'DatabaseClasses/symbDBClass.html' && Example Source for User-Defined Database Class X

Example Source for A Fancier Database Class

X The class symbDB and related methods described here provide an implementation of S user-defined database classes, somewhat more sophisticated and user-friendly than the dumpDataBase class. That class simply wrote objects to files with the same name, which inherently restricts the names of the objects. In this implementation the database software maintains a separate file for the vector of current object names. This file produces a hash table (in the form of a string vector) used to maintain a parallel table of the contents of the database. The position of a name in the string vector is encoded as the file name. To make this work easily over removals of objects, empty strings are kept where objects are removed. The name for a first-time assignment replaces an empty string or goes on the end of the string vector. In directory databases, mostly from old tradition, S uses object names as file names when these are legal and otherwise constructs a table of maps to true file names. In fact the strategy used here would work fine for directory databases as well.

The symbDB class is a slightly more typical example than its simpler counterpart, in that the code must worry a little about the consistency of the explicit table in the string object and the database dictionary kept by the evaluator manager. Suppose an error takes place in writing a file, for example: what happens to the two tables? The implementation shown here does only a minimal amount of bullet-proofing against such problems. X

Class Definition

X Defines the class to extend the virtual class "database" and to contain one slot, for the string path name of the corresponding directory. X
    setClass("symbDB",
X        representation("database", path = "character"))
X

Generator Function for the Class

X Takes as argument a character string for the directory's path and, optionally, a flag to say whether to create the directory if it does not exist. X
"symbDB" =
X  ## make an object of class `"symbDB"' corresponding to the directory
X  ## `path'.  The flag `create' allows or suppresses creating the directory if it
X  ## does not exist (by default, the user is asked, if S is in interactive mode).
X  function(path,
X           create = identical(dialog(paste("Directory \"", path,
X             "\"for database does not exist; create?", sep=""), c("y", "n"), 2),1))
{
X  path = as(path, "character")
X  if(length(path)!=1) stop("needed a single character string for path")
X  if(!isDirectory(path)) {
X    if(create) {
X      shell(paste("mkdir", path), mustWork=T)
X      file = paste(path, "objects", sep="/")
X      cat("", file=file, sep="")
X    }
X    else
X      warning("Directory \"", path, "\"for database does not exist; path validity not checked")
X  }
X  if(isDirectory(path))
X    ## get a fully-qualified path
X    path = shell(paste("cd", path, "; pwd"))
X  new("symbDB", path = path)
}
X
X X X

Method for dbobjects: Listing the Object Names

X This method returns a character vector of the current object names in the directory. It is normally only called when the database is being attached. From then on, the S evaluation manager maintains its own table of the contents of the database. As with ordinary directory databases, this means that multiple processes using the same database simultaneously are not automatically co-ordinated.

For this particular database class, the sdbObjects utility function actually accesses the information about current database contents, and returns the appropriate character vector.

setMethod("dbobjects", "symbDB", 
function(database)
{
X  ans = sdbObjects(database)
X  ## may be empty strings corresponding to deleted objects
X  sort(ans[nchar(ans) > 0])
}
)
X

Method for dbread: Read an Object

X Given the database and the name of an object guaranteed to be in the datbase, this method reads it in. It uses the sdbNameToFile utility in this package to get the name of the file containing the object. Since all objects are written in the standard S symbolic dump format by dataPut, this method just calls dataGet to do the reading. X
setMethod("dbread", "symbDB", 
function(database, name)
{
X  path = sdbNameToFile(database, name)
X  if(nchar(path)==0)
X    stop("object \"", name, "\" not in database")
X  dataGet(path)
}
)
X The check on path is not needed in normal usage, but it is possible for dataGet to fail, for example if some other process is concurrently using the same directory and removes the object. X

Method for dbwrite: Write an Object

X This method takes the database, an object to write out, and the name the object should have when assigned in the database. The local utility sdbNameToFile is called to get the path for the corresponding file, and is told to create a new path if one does not exist. Once this is done, all that remains is to call the dataPut function to write the object in standard symbolic dump format to the corresponding file.
setMethod("dbwrite", "symbDB", 
function(database, name, object) {
X  path = sdbNameToFile(database, name, T)
X  dataPut(object, path)
}
)
X

Method for dbremove: Remove an Object

X This method removes the name from the list of names in the database, and removes the corresponding file containing the object as dumped out. The name is erased from the character vector supplied by sdbObjects; an empty string is stored in its place, to allow the file name to be reused for a later assignment. The utility setSdbObjects stores the revised vector of names both in the session and externally in the directory. The file identified by sdbNameToFile is also removed. Notice that we have to get the file name before modifying the session's view of what objects exist in the database. X
setMethod("dbremove", "symbDB",
X          ## removes the name from the list of objects, saves the updated list of objects,
X          ## and then removes the corresponding file from the directory.
function(database, name) {
X  objects = sdbObjects(database)
X  i = match(name, objects)
X  if(is.na(i))
X    stop("bad call to dbremove method: \"",
X         name, "\" is not in this database")
X  file = sdbNameToFile(database, name)
X  if(i == length(objects))
X    length(objects) = i-1
X  else el(objects, i) = ""
X  sdbSetObjects(database, objects)
X  shell(paste("rm", file))
}
)
X

Some Utility Functions

X These functions use and manipulate the internal session object that defines the contents of the database. This object has a constructed name equivalent to path and stored in the session database. X X
John Chambers<jmc@research.bell-labs.com>
Last modified: Fri Oct 2 16:53:34 EDT 1998 SHAR_EOF $shar_touch -am 1002165398 'DatabaseClasses/symbDBClass.html' && chmod 0664 'DatabaseClasses/symbDBClass.html' || $echo 'restore of' 'DatabaseClasses/symbDBClass.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'DatabaseClasses/symbDBClass.html:' 'MD5 check failed' 2df4e0c3d647c423b2817d5705ab2842 DatabaseClasses/symbDBClass.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'DatabaseClasses/symbDBClass.html'`" test 7748 -eq "$shar_count" || $echo 'DatabaseClasses/symbDBClass.html:' 'original size' '7748,' 'current size' "$shar_count!" fi fi # ============= DatabaseClasses/ttt.html ============== if test -f 'DatabaseClasses/ttt.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'DatabaseClasses/ttt.html' '(file already exists)' else $echo 'x -' extracting 'DatabaseClasses/ttt.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'DatabaseClasses/ttt.html' && User Defined Database Classes X X X X

User-Defined Database Classes

X

Database Classes

Section 5.3 of Programming with Data mentions a number of different ways to define an S database, including user-defined database classes. Not much more is said about these in the book.

Here they are. The idea quite powerful; although a moderate amount of programming is needed to add a new kind of database class, the results can usually be reused for other, similar applications. This page outlines what is needed, and gives an example using the S symbolic dump format. The example has some mild practicality, in providing a portable database that could be used in a network file system over diverse hardware. It's main virtue, though, is to work out the essential ideas in a context that doesn't require discussing any other software, such as database management systems.

A database class is any class that extends the virtual class database. Attaching an object from this class causes S to look for corresponding methods for a set of basic database functions, to read, write, remove objects. Once the required set of such methods is defined, you can attach an object of this class and use the resulting database like any other.

Notice that this is different from attaching a list-like object as a database. In that case, all the data resided in the single object. For database classes, on the other hand, the database object is usually just some identifier for a directory, database, URL or whatever. It's just there to tell the evaluator to use corresponding methods to deal with the data.

Let's do an example. In this case we are going to use a directory in the file system as our database. But instead of relying on the standard S binary representation for objects, we will read and write files using the text-only "symbolic dump format" (pp 237-241). The class just needs to extend database and contain the path for the directory.

symbolicDataBase.class.S

The design of the database class will be simple in the extreme: a directory will contain all the objects, with one file, "objects", being a list of the names of the objects in the database, one per line, and the corresponding objects being stored in files ".N", with N corresponding to the line number.

With this definition in mind, we can write a simple generating function that creates the directory and initializes it.

The next step is to write some methods for the database operations. X SHAR_EOF $shar_touch -am 0903170598 'DatabaseClasses/ttt.html' && chmod 0664 'DatabaseClasses/ttt.html' || $echo 'restore of' 'DatabaseClasses/ttt.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'DatabaseClasses/ttt.html:' 'MD5 check failed' 24fe0da32792218e9bef6e86be688312 DatabaseClasses/ttt.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'DatabaseClasses/ttt.html'`" test 3031 -eq "$shar_count" || $echo 'DatabaseClasses/ttt.html:' 'original size' '3031,' 'current size' "$shar_count!" fi fi # ============= Documentation/FutureProjects.html ============== if test ! -d 'Documentation'; then $echo 'x -' 'creating directory' 'Documentation' mkdir 'Documentation' fi if test -f 'Documentation/FutureProjects.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/FutureProjects.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/FutureProjects.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/FutureProjects.html' && Future Directions for S Documentation X

Future Developments

We have made a significant effort to modularize the documentation from the core S system (at the minimal expense of rarely noticed used inefficiency.) The current pipeline for the processing allows other X X
Duncan Temple Lang<duncan@research.bell-labs.com>
Last modified: Mon Apr 20 11:07:37 EDT 1998 SHAR_EOF $shar_touch -am 0420135198 'Documentation/FutureProjects.html' && chmod 0664 'Documentation/FutureProjects.html' || $echo 'restore of' 'Documentation/FutureProjects.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/FutureProjects.html:' 'MD5 check failed' 0653c63ac2adaa53ec3ff560edc07d15 Documentation/FutureProjects.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/FutureProjects.html'`" test 1040 -eq "$shar_count" || $echo 'Documentation/FutureProjects.html:' 'original size' '1040,' 'current size' "$shar_count!" fi fi # ============= Documentation/Overview.html ============== if test -f 'Documentation/Overview.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/Overview.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/Overview.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/Overview.html' && Overview of the S Documentation Engine X

Overview of the S Documentation Engine

We briefly describe the different stages in the life of an S documentation object and the associated tools for each phase. This material is covered in more detail in Chapter 9 of Programming with Data. The steps below show how to create a documentation object and provide programmers with the necessary background information to customize the output. X

Generating the Document

In most cases, we create the initial documentation object using the self-documenting feature in S4. We do this with the S expression
dumpDoc("topic")
where topic is usually the name of a function. See the book or the online documentation of dumpDoc for options. This creates a file containing the documentation provided in the object and the default fields the developer may complete. X

Editing the Document

Arbitrary text can be inserted into the elements of the document using a standard editor. We use emacs (currently with html-helper-mode) to add HTML-like elements. The permissible SGML elements are described in Writing Documentation X

Creating the S object

To create the S documentation object in the S system from the ASCII input file, give the command
sourceDoc('file-name')
X

Converting Documentation to Target Format

Currently, the documentation object can be converted into SGML, HTML and plain text representations. However, adding new targets such as TeXinfo, TeX/LaTeX, man pages, etc. is also conceptually easy. The first is trivial, involving only printing of the document - show(getDoc('object-name'). The other conversions involve translating the SGML elements into `commands' in the target language. The functions printDoctargetname() are used to do this conversion and utilize a named table of translation mappings. X

X To create mappings for a new target, one can simply copy the object textMappings and edit its contents as needed. If new elements/tags are added to documentation structure, corresponding entries should be added to each of the top-level mapping objects. X


Duncan Temple Lang<duncan@research.bell-labs.com>
Last modified: Fri Jun 19 10:45:07 EDT 1998 SHAR_EOF $shar_touch -am 0619104598 'Documentation/Overview.html' && chmod 0664 'Documentation/Overview.html' || $echo 'restore of' 'Documentation/Overview.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/Overview.html:' 'MD5 check failed' 45c0b9d92aeed4d71aef65dc40730394 Documentation/Overview.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/Overview.html'`" test 2789 -eq "$shar_count" || $echo 'Documentation/Overview.html:' 'original size' '2789,' 'current size' "$shar_count!" fi fi # ============= Documentation/Programming.html ============== if test -f 'Documentation/Programming.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/Programming.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/Programming.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/Programming.html' && Programming with the S Documentation System X

Programming with the S Documentation System

X X

S Documentation Objects

X S documentation objects are hierarchically organized, largely around elements of class "sgmlTag", the S analog of an arbitrary tag in the SGML language. These sgmlTag objects contain a tag name (a character string), attributes and a body. The attributes, if not empty, is a named object with character strings as the data elements. The body is a list, containing further sgmlTag objects and/or plain text. The recursive structure of the objects means that arbitrary SGML structure can be represented in the documentation object, allowing transformation in either direction without loss of information.

Information about rendering the documentation objects is not (wherever possible) stored in the documentation objects themselves. Instead, a (potentially extensible) set of mapping table objects contain programmatic rendering or processing information for S computations on documentation objects, and DTD tables and other non-S mechanisms can be used to control rendering or other processing once the documentation objects have been converted to SGML. X X

Converting to Text

X

The Conversion Model

X The fundamental output operation in the S documentation system is to convert all or part of documentation objects to text, typically to be written to an S connection. Depending on why the conversion is being done, the form of the output will be driven by S objects, known as mapping tables. The basic function involved is
sgmlTagToText(object, maps, level = 0)
This function takes an S object, usually of class sgmlTag and a mapping table object, and returns text corresponding to the object. The function looks up the tag name of the object in a mapping table. It can find a character vector, an S function or expression, a special object of class "sgmlMap", or nothing (a null object).

What happens to the object in these cases is as follows:

X
expression or function X
This expression is evaluated in a new frame X which is constructed to contain local variables X for the different components of the object being processed X in processSgmlMap and the mapping table and specific map. X X
character string X
The character string is returned as the corresponding output. X For example, this is how section headings are defined in turning X the documentation object into plain text for local printing. X
An object of class sgmlMap. X
This can contain a prefix and a suffix X and a main slot which can be X any legitimate map object (including X an sgmlMap object itself). X
NULL object X
No special action for this tag, but process the body of the X object, which can itself contain sgmlTag objects.
The result of evaluating the map object can be either a string or a list containing the text and the resulting object. This allows processing of the entire object and its sub-components in one step (e.g. s-usage tag) and removing the children to avoid duplicate processing. X

Mapping Tables

The translation mapping table is a hierarchical structure with elements for the different documentation types. These types mirror the DTD structure in which the more specific structures include the less specific ones. X X X X X X
generalElements that can typically be used in any part of any document.
doc Top-level sections (e.g. X s-title, etc.) X and basic information for a document (version, etc.) X
functionSections and structures for a document describing an S X function. X
classSections and structures for a document describing an S class.
method Sections and structures for a document describing an S X method. This is a simple extension of function above. X
Each of these components is also a named list where the names of the elements are the SGML tag names (e.g. s-expression, code, s-section, etc.) and the elements are the objects to be returned when this tag name is matched during the conversion of an sgmlTag object to text. X

SGML Tags Defined in the Mapping Table

X The following table lists the SGML tags programmed in the standard mapping tables. Those found in the "general" section are included in "doc", and both these will be included in "function" and "class". Finally, "method" extends "function". X X
Tag NameFound InSection or Sub-section?
"application" general No
"br" general No
"code" general No
"directory" general No
"file" general No
"p" general No
"program" general No
"s-arg" function Sub
"s-args" function Yes
"s-args-optional" function Yes
"s-args-require" function Yes
"s-argument" function No
"s-author" doc Yes
"s-background" doc Yes
"s-bugs" doc Yes
"s-call" function No
"s-class" general Yes
"s-class-slot" class Yes
"s-class-title" class Yes
"s-description" doc Yes
"s-details" doc Yes
"s-docclass" doc Yes
"s-example" doc No
"s-examples" doc Yes
"s-expression" general No
"s-extends" class Yes
"s-extends-class" class Yes
"s-function-name" function No
"s-keywords" doc Yes
"s-note" doc Yes
"s-object" general No
"s-prototype-element" class No
"s-references" doc Yes
"s-section" doc Yes
"s-see" doc Yes
"s-side-effects" function Yes
"s-signature" method Yes
"s-signature-arg" method Sub
"s-slots" class Yes
"s-title" doc Yes
"s-topics" doc Yes
"s-usage" function Yes
"s-usage-expression" function Yes
"s-value" doc Yes
"s-warning" doc No
"s-warnings" doc Yes
"url" general No
Documentation Tags
X X
Duncan Temple Lang<duncan@research.bell-labs.com>
Last modified: Fri Jun 19 16:55:55 EDT 1998 SHAR_EOF $shar_touch -am 0619165598 'Documentation/Programming.html' && chmod 0664 'Documentation/Programming.html' || $echo 'restore of' 'Documentation/Programming.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/Programming.html:' 'MD5 check failed' d96e153183754d190846d65c40dac883 Documentation/Programming.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/Programming.html'`" test 8126 -eq "$shar_count" || $echo 'Documentation/Programming.html:' 'original size' '8126,' 'current size' "$shar_count!" fi fi # ============= Documentation/Rendering.html ============== if test -f 'Documentation/Rendering.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/Rendering.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/Rendering.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/Rendering.html' && Rendering S Documentation X

Rendering S Documentation

X X X

Style-Sheets

X X
Duncan Temple Lang<duncan@research.bell-labs.com>
Last modified: Mon Apr 20 09:38:34 EDT 1998 SHAR_EOF $shar_touch -am 0420135198 'Documentation/Rendering.html' && chmod 0664 'Documentation/Rendering.html' || $echo 'restore of' 'Documentation/Rendering.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/Rendering.html:' 'MD5 check failed' 6c83f2a2be2e67e56fce1678da989025 Documentation/Rendering.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/Rendering.html'`" test 460 -eq "$shar_count" || $echo 'Documentation/Rendering.html:' 'original size' '460,' 'current size' "$shar_count!" fi fi # ============= Documentation/WritingDocumentation.html ============== if test -f 'Documentation/WritingDocumentation.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/WritingDocumentation.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/WritingDocumentation.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/WritingDocumentation.html' && Writing S Documentation X

Writing S Documentation

X The S documentation, when mapped into SGML, uses some special SGML tags (in addition to the tags normally allowed in HTML documents). These tags relate to the structure of the documentation, that is, to its meaning in terms of what it describes, of particular concepts in S, etc. In contrast, the rendering decisions are largely independent of these tags, and are defined by style sheets and/or other tags included in the document. X

Top-Level Structure

X The documentation object is structured as sections (effectively, named elements of the object). A number of sections are pre-defined, depending on the documentation class (e.g, "function" or "class"). Other arbitrary sections can be added.

S Expressions as SGML

X The documentation library in S includes the ability to map an arbitrary S expression into an SGML structure that reflects the S object. In other words, instead of just producing a text representation of the expression, this approach reproduces the full structure of the S expression. You can benefit from this structure if you want to do computations that understand something about that structure (e.g., treating particular S syntactic structures such as for loops in a special way). Most of the time, you don't want to edit the resulting SGML directly, but instead to provide the S object to an S function that generates the SGML.

Tags Generated from S Expression Objects
Tag Class(es)
"s-assign-expr" "=", "<-", "<<-"
"s-for-expr" "for"
"s-while-expr" "while"
"s-repeat-expr" "repeat"
"s-if-expr" "if"
"s-paren-expr" "("
"s-brace-expr" "{"
"s-name-expr" "name"
"s-function-expr" "function"
"s-formalArgs-expr" Any
"s-formalArg-expr" Any
"s-call-expr" "call"
"s-unaryOp-expr" "call"
"s-binaryOp-expr" "call"
"s-arg-expr" Any
"s-literal-expr" Any non-language
X X


Duncan Temple Lang<duncan@research.bell-labs.com>
Last modified: Fri Jun 19 15:27:57 EDT 1998 SHAR_EOF $shar_touch -am 0619154698 'Documentation/WritingDocumentation.html' && chmod 0664 'Documentation/WritingDocumentation.html' || $echo 'restore of' 'Documentation/WritingDocumentation.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/WritingDocumentation.html:' 'MD5 check failed' 3fe7c3feae8728320351a5517a4c2bce Documentation/WritingDocumentation.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/WritingDocumentation.html'`" test 2911 -eq "$shar_count" || $echo 'Documentation/WritingDocumentation.html:' 'original size' '2911,' 'current size' "$shar_count!" fi fi # ============= Documentation/index.html ============== if test -f 'Documentation/index.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/index.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/index.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/index.html' && S Documentation Engine X

The S Documentation System

As described in Chapter 9 of Programming with Data, online documentation in S uses S objects, kept in a special meta database. Computations for online help are a combination of S computations on these objects and other computations using a mapping of documentation objects to the SGML language. The advantage of this organization is that both the S and the SGML versions of the documentation are capable of representing the structure of the information. The S side of the computations can map information from general S objects into the documentation (for example, using objects that represent S expressions), while the SGML side can organize both structural information about the document (what information is allowed and where) and also rendering information when web-based tools are used to view the documentation. X The use of SGML allow users to exploit the new XML technology currently being developed. Among other benefits, we should see simpler and more productive search methods operating on the documentation and opportunities for research on more effective integrated help systems. X X X X X X
Duncan Temple Lang<duncan@research.bell-labs.com>
Last modified: Fri Jun 19 10:23:15 EDT 1998 SHAR_EOF $shar_touch -am 0619154898 'Documentation/index.html' && chmod 0664 'Documentation/index.html' || $echo 'restore of' 'Documentation/index.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/index.html:' 'MD5 check failed' c61ac19e513d83b473d6b1a53592a49b Documentation/index.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/index.html'`" test 2003 -eq "$shar_count" || $echo 'Documentation/index.html:' 'original size' '2003,' 'current size' "$shar_count!" fi fi # ============= Documentation/links.html ============== if test -f 'Documentation/links.html' && test "$first_param" != -c; then $echo 'x -' SKIPPING 'Documentation/links.html' '(file already exists)' else $echo 'x -' extracting 'Documentation/links.html' '(text)' sed 's/^X//' << 'SHAR_EOF' > 'Documentation/links.html' && SGML/XML Links X

Related SGML/XML Links

X
SGML home page X
Information on SGML. See the software X section for a plethora of useful tools. X
W3 page X
General information on Web-related standards and technology. X Specifically, see the XML page. X X
Jim Clark's Home page X
A publically available SGML parser (with source code) X used as the basis of our system. X
Editing SGML mode for emacs. X
PSGML X is an major-mode for Emacs for editing SGML documents and DTD X files. We will prepare bindings for the S-specific elements/tags X for this and ESS X editing mode. X Also, html-helper-mode X is a mode for editing HTML documents with support for X customization the known tags, etc. X
Emacs mode X for editing DTDs X
This contains support for editing DTDs in emacs and X an Elisp function for X generating a TAGS file for DTDs for easy location of X definitions within DTDs. X (Note that this function is currently case sensitive X and ENTITY and ELEMENT keywords must be capitalized X in order to be recognized in the command.) X But then Esc-. <tag-name> will jump to the spot. X
X X
Duncan Temple Lang <duncan@research.bell-labs.com>
Last modified: Fri Oct 2 18:02:43 EDT 1998 SHAR_EOF $shar_touch -am 0617175998 'Documentation/links.html' && chmod 0664 'Documentation/links.html' || $echo 'restore of' 'Documentation/links.html' 'failed' if ( md5sum --help 2>&1 | grep 'sage: md5sum \[' ) >/dev/null 2>&1 \ && ( md5sum --version 2>&1 | grep -v 'textutils 1.12' ) >/dev/null; then md5sum -c << SHAR_EOF >/dev/null 2>&1 \ || $echo 'Documentation/links.html:' 'MD5 check failed' 3985f5a1936bae2cc3f24dd981485d2a Documentation/links.html SHAR_EOF else shar_count="`LC_ALL= LC_CTYPE= LANG= wc -c < 'Documentation/links.html'`" test 2080 -eq "$shar_count" || $echo 'Documentation/links.html:' 'original size' '2080,' 'current size' "$shar_count!" fi fi rm -fr _sh02895 exit 0