SketchToPaperUI.java

00001 package edu.stanford.hci.r3.tools.design.sketch;
00002 
00003 import java.awt.geom.Point2D;
00004 import java.awt.geom.Rectangle2D;
00005 import java.io.File;
00006 import java.io.FileNotFoundException;
00007 import java.io.FileOutputStream;
00008 import java.io.IOException;
00009 import java.io.PrintStream;
00010 import java.text.DecimalFormat;
00011 import java.util.ArrayList;
00012 import java.util.List;
00013 import java.util.regex.Matcher;
00014 import java.util.regex.Pattern;
00015 
00016 import edu.stanford.hci.r3.PaperToolkit;
00017 import edu.stanford.hci.r3.pen.Pen;
00018 import edu.stanford.hci.r3.pen.PenSample;
00019 import edu.stanford.hci.r3.pen.batch.PenSynch;
00020 import edu.stanford.hci.r3.pen.handwriting.HandwritingRecognitionService;
00021 import edu.stanford.hci.r3.pen.ink.Ink;
00022 import edu.stanford.hci.r3.pen.ink.InkStroke;
00023 import edu.stanford.hci.r3.pen.ink.InkUtils;
00024 import edu.stanford.hci.r3.pen.streaming.listeners.PenListener;
00025 import edu.stanford.hci.r3.tools.design.util.Regions;
00026 import edu.stanford.hci.r3.util.DebugUtils;
00027 import edu.stanford.hci.r3.util.files.FileUtils;
00028 
00040 public class SketchToPaperUI {
00041 
00045         private static void testTranslateXMLFile() {
00046                 String fileName = // "penSynch/data/XML/2007_03_24__19_51_50_AJ3-AAA-ZU3-7X.xml";
00047                 "penSynch/data/XML/2007_03_10__01_09_38_SketchedPaperUI.xml";
00048                 new SketchToPaperUI().translate(new File(fileName), "SketchedPaperUI", new File("."));
00049         }
00050 
00051         private Pen pen;
00052 
00056         public SketchToPaperUI() {
00057                 DebugUtils.println("New Sketch To Paper UI");
00058 
00059                 // set up a pen, and a pen listener...
00060                 pen = new Pen();
00061                 pen.startLiveMode();
00062                 pen.addLivePenListener(new PenListener() {
00063                         @Override
00064                         public void penDown(PenSample sample) {
00065                                 
00066                         }
00067 
00068                         @Override
00069                         public void penUp(PenSample sample) {
00070                                 
00071                         }
00072 
00073                         @Override
00074                         public void sample(PenSample sample) {
00075                                 
00076                         }
00077                 });
00078         }
00079 
00080         // private static void highlight(List<InkStroke> strokes) {
00081         //
00082         // StringBuilder sb = new StringBuilder();
00083         // sb.append("<highlight>");
00084         // for (InkStroke s : strokes) {
00085         // sb.append("<stroke begin=\"" + s.getFirstTimestamp() + "\"/>\n");
00086         // }
00087         // sb.append("</highlight>");
00088         // FileUtils
00089         // .writeStringToFile(sb.toString(), new File("flash/data/highlightTheseStrokes.xml"));
00090         // }
00091 
00097         public void addPenListener(PenListener penListener) {
00098                 pen.addLivePenListener(penListener);
00099         }
00100 
00104         public void exit() {
00105                 DebugUtils.println("Exiting Sketch To Paper UI");
00106                 pen.stopLiveMode();
00107         }
00108 
00115         public void translate(File strokeFile, String className, File outputFolder) {
00116                 PenSynch penSynch = new PenSynch(strokeFile);
00117                 List<Ink> importedInk = penSynch.getImportedInk();
00118 
00119                 // TODO: some kind of intelligent grouping of strokes that end near
00120                 // each other
00121 
00122                 // Biggest stroke becomes the sheet
00123                 InkStroke biggestStroke = InkUtils.getStrokeWithLargestArea(importedInk);
00124 
00125                 // Strokes inside sheet are regions
00126                 List<InkStroke> regionStrokes = InkUtils.getAllStrokesContainedWithin(importedInk, biggestStroke);
00127 
00128                 // Strokes that overlap the sheet but go outside are connectors
00129                 List<InkStroke> connectors = InkUtils.getStrokesPartlyOutside(importedInk, biggestStroke);
00130 
00131                 // Strokes outside the sheet are events
00132                 List<InkStroke> outsideStrokes = InkUtils.getAllStrokesOutside(importedInk, biggestStroke);
00133                 // Cluster events (since they're words)
00134                 List<Ink> events = InkUtils.clusterStrokes(outsideStrokes, 2);
00135 
00136                 // Start the recognizer
00137                 HandwritingRecognitionService service = HandwritingRecognitionService.getInstance();
00138 
00139                 // Calculate the size of the sheet in inches (make it fit in 8.5x11)
00140                 Rectangle2D sheet = biggestStroke.getBounds();
00141                 double scale = Regions.makeItFit(sheet.getWidth(), sheet.getHeight(), 8.5, 11);
00142 
00143                 // Print out to...
00144                 PrintStream outXML = null;
00145                 PrintStream outJava = null;
00146                 try {
00147                         outXML = new PrintStream(new FileOutputStream(new File(outputFolder, className + ".xml")));
00148                         outJava = new PrintStream(new FileOutputStream(new File(outputFolder, className + ".java")));
00149                 } catch (FileNotFoundException e1) {
00150                         e1.printStackTrace();
00151                 }
00152 
00153                 // Number format...
00154                 DecimalFormat df = new DecimalFormat("0.###");
00155 
00156                 outXML.println("<sheet width=\"" + df.format(sheet.getWidth() * scale) + "\" height=\""
00157                                 + df.format(sheet.getHeight() * scale) + "\">");
00158 
00159                 class Region {
00160                         // InkStroke stroke = null;
00161                         Rectangle2D bounds = null;
00162                         String eventType = null;
00163                         String name = null;
00164                 }
00165 
00166                 List<Region> regions = new ArrayList<Region>();
00167 
00168                 // Print out regions
00169                 int strokeId = 1;
00170                 for (InkStroke region : regionStrokes) {
00171                         Region r = new Region();
00172                         regions.add(r);
00173                         // r.stroke = region;
00174                         r.bounds = region.getBounds();
00175                         InkStroke c = null;
00176                         // Find the connection that matches this region
00177                         for (InkStroke connection : connectors) {
00178                                 if (connection.getBounds().intersects(r.bounds)) {
00179                                         c = connection;
00180                                         break;
00181                                 }
00182                         }
00183                         Ink event = null;
00184 
00185                         if (c != null) {
00186 
00187                                 // Find the endpoint outside of the region
00188                                 PenSample p = c.getStart();
00189                                 if (r.bounds.contains(p.getX(), p.getY()))
00190                                         p = c.getEnd();
00191 
00192                                 // Find the event nearest that endpoint
00193                                 event = InkUtils.getInkNearPoint(events, new Point2D.Double(p.getX(), p.getY()), 40.0);
00194 
00195                                 // If an event is found...
00196                                 if (event != null) {
00197                                         // Recognize the text...
00198                                         String result = service.recognizeHandwriting(event
00199                                                         .getAsXML(false /* no separator lines */));
00200 
00201                                         // Split it on non-alpha numeric characters
00202                                         String pieces[] = result.split("[^a-zA-Z0-9]");
00203                                         // Second piece is name
00204                                         if (pieces.length > 1)
00205                                                 r.name = pieces[1];
00206                                         // First is event type
00207                                         r.eventType = pieces[0].toLowerCase();
00208                                         // TODO: there is probably a better way to split this
00209                                 }
00210                         }
00211 
00212                         // If we got no name, auto-generate one (this is problematic if you
00213                         // revise your sketch, since they could potentially be renumbered)
00214                         if (r.name == null)
00215                                 r.name = "region" + (strokeId++);
00216 
00217                         r.name = Character.toUpperCase(r.name.charAt(0)) + r.name.substring(1);
00218 
00219                         // Print it out
00220                         outXML.print("  <region name=\"" + r.name + "\" x=\""
00221                                         + df.format((r.bounds.getX() - sheet.getX()) * scale) + "\" y=\""
00222                                         + df.format((r.bounds.getY() - sheet.getY()) * scale) + "\" width=\""
00223                                         + df.format(r.bounds.getWidth() * scale) + "\" height=\""
00224                                         + df.format(r.bounds.getHeight() * scale) + "\"");
00225                         if (event != null) {
00226                                 outXML.println(">");
00227                                 outXML.println("   <eventHandler type=\"" + r.eventType + "\"/>");
00228                                 outXML.println("  </region>");
00229                         } else {
00230                                 outXML.println("/>");
00231                         }
00232                 }
00233                 outXML.println("</sheet>");
00234                 outXML.close();
00235 
00236                 // The Template for creating a Paper UI Class...
00237                 String template = FileUtils.readFileIntoStringBuffer(
00238                                 PaperToolkit.getResourceFile("/designer/template.txt")).toString();
00239 
00240                 template = template.replace("{CLASSNAME}", className);
00241 
00242                 // Find patterns of type {REPEAT:REGIONS} ... {/REPEAT:REGIONS}
00243                 // in the template
00244                 Matcher repeatMatcher = Pattern.compile("\\{REPEAT:REGIONS\\}([\\s\\S]*?)\\{/REPEAT:REGIONS\\}",
00245                                 Pattern.MULTILINE | Pattern.CASE_INSENSITIVE).matcher(template);
00246                 System.out.println("Matches = " + repeatMatcher.matches());
00247 
00248                 int lastPosition = 0;
00249 
00250                 while (repeatMatcher.find()) {
00251                         // Print text before the repeat region
00252                         outJava.print(template.substring(lastPosition, repeatMatcher.start()));
00253 
00254                         // Loop through regions...
00255                         for (Region region : regions) {
00256 
00257                                 String repeatString = repeatMatcher.group(1).replace("{REGION.NAME}", region.name);
00258 
00259                                 // Find {IF:XX} ... {/IF:XX} blocks in the repeat region
00260 
00261                                 Matcher ifMatcher = Pattern.compile("\\{IF:([A-Z]+)\\}([\\s\\S]*?)\\{/IF:([A-Z]+)\\}",
00262                                                 Pattern.MULTILINE | Pattern.CASE_INSENSITIVE).matcher(repeatString);
00263 
00264                                 int lastPosition2 = 0;
00265 
00266                                 while (ifMatcher.find()) {
00267                                         // print text before {IF}
00268                                         outJava.print(repeatString.substring(lastPosition2, ifMatcher.start()));
00269 
00270                                         String type = ifMatcher.group(1);
00271                                         String ifString = ifMatcher.group(2);
00272 
00273                                         // If this event is valid for this region, print out the
00274                                         // if block
00275                                         if (type.toLowerCase().equals(region.eventType))
00276                                                 outJava.print(ifString);
00277                                         lastPosition2 = ifMatcher.end();
00278                                 }
00279                                 // print text after {/IF}
00280                                 outJava.print(repeatString.substring(lastPosition2));
00281                         }
00282                         lastPosition = repeatMatcher.end();
00283                 }
00284                 // print text after the region
00285                 outJava.print(template.substring(lastPosition));
00286 
00287                 outJava.close();
00288 
00289                 // Close down recognizer
00290                 service.exitServer();
00291 
00292                 // Ron's notes...
00293                 // <sheet width="8.5" height="11">
00294                 // <region name="submit" type="button" x="6.5" y="9" w="1.5" h="1">
00295                 // <eventHandler type="click" name="onSubmit"/>
00296                 // </region>
00297                 // <region name="logo" type="image" src="files/logo.png"/>
00298                 // <region name="inkCapture" type="capture">
00299                 // <eventHandler type="inkWell" name="onInkArrived"/>
00300                 // </region>
00301                 // </sheet>
00302                 //
00303                 // OR, use PaperToolkit.showMe(insideStroke); // this colors the stroke in an external
00304                 // view...
00305                 // it renders an XML file of the INK with some cool colors. =)
00306                 // then it opens the browser to the right HTML page with the right query string. =)
00307                 // Find strokes that go from within the sheet to outside the sheet
00308                 // These are event handlers
00309                 // xxx, at each step, remove the strokes from consideration! =)
00310                 // write about this algorithm in the paper!
00311                 // Find the next near the endpoints of these event handlers.
00312                 // Do handwriting recognition on them.
00313                 // Create Event Handlers!
00314                 // Write about this!!! =)
00315                 // Part II...
00316                 // A second approach is more of a connected components approach?
00317                 // Take a look at Andy Wilson's?
00318                 // Close off strokes, and find buttons and such...
00319         }
00320 }

Generated on Sat Apr 14 18:21:38 2007 for R3 Paper Toolkit by  doxygen 1.4.7