using System; using System.Drawing; using System.Collections; using System.ComponentModel; using System.Windows.Forms; using System.Data; //for Matrix using System.Drawing.Drawing2D; //for Ink using Microsoft.Ink; //Speech using SpeechLib; namespace tabletStrator { /// /// Summary description for Form1. /// public class TStrator : System.Windows.Forms.Form { private AxMediaPlayer.AxMediaPlayer axMediaPlayer1; private System.Windows.Forms.StatusBar statusBar1; private Microsoft.Ink.InkPicture inkPicture1; private System.Windows.Forms.Timer timer1; private System.Windows.Forms.ColorDialog colorDialog1; private System.Windows.Forms.Button clrBut; private System.Windows.Forms.Button colBut; private System.Windows.Forms.TrackBar penTrack; private System.Windows.Forms.OpenFileDialog openFileDialog1; private System.Windows.Forms.Button openBut; private System.Windows.Forms.Button erBut; private System.Windows.Forms.GroupBox gestBox; private System.Windows.Forms.Button ccBut; private System.Windows.Forms.Button spchBut; private System.Windows.Forms.Button ttsBut; private System.ComponentModel.IContainer components; public TStrator() { // // Required for Windows Form Designer support // InitializeComponent(); // // TODO: Add any constructor code after InitializeComponent call // } /// /// Clean up any resources being used. /// protected override void Dispose( bool disposing ) { if(timer1.Enabled == true) timer1.Enabled = false; if(axMediaPlayer1 != null) axMediaPlayer1 = null; if( disposing ) { if (components != null) { components.Dispose(); } } base.Dispose( disposing ); } #region Windows Form Designer generated code /// /// Required method for Designer support - do not modify /// the contents of this method with the code editor. /// private void InitializeComponent() { this.components = new System.ComponentModel.Container(); System.Resources.ResourceManager resources = new System.Resources.ResourceManager(typeof(TStrator)); this.axMediaPlayer1 = new AxMediaPlayer.AxMediaPlayer(); this.statusBar1 = new System.Windows.Forms.StatusBar(); this.inkPicture1 = new Microsoft.Ink.InkPicture(); this.timer1 = new System.Windows.Forms.Timer(this.components); this.colorDialog1 = new System.Windows.Forms.ColorDialog(); this.clrBut = new System.Windows.Forms.Button(); this.colBut = new System.Windows.Forms.Button(); this.penTrack = new System.Windows.Forms.TrackBar(); this.openFileDialog1 = new System.Windows.Forms.OpenFileDialog(); this.openBut = new System.Windows.Forms.Button(); this.erBut = new System.Windows.Forms.Button(); this.gestBox = new System.Windows.Forms.GroupBox(); this.ccBut = new System.Windows.Forms.Button(); this.spchBut = new System.Windows.Forms.Button(); this.ttsBut = new System.Windows.Forms.Button(); ((System.ComponentModel.ISupportInitialize)(this.axMediaPlayer1)).BeginInit(); ((System.ComponentModel.ISupportInitialize)(this.penTrack)).BeginInit(); this.SuspendLayout(); // // axMediaPlayer1 // this.axMediaPlayer1.Anchor = (((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) | System.Windows.Forms.AnchorStyles.Left) | System.Windows.Forms.AnchorStyles.Right); this.axMediaPlayer1.Location = new System.Drawing.Point(0, 48); this.axMediaPlayer1.Name = "axMediaPlayer1"; this.axMediaPlayer1.OcxState = ((System.Windows.Forms.AxHost.State)(resources.GetObject("axMediaPlayer1.OcxState"))); this.axMediaPlayer1.Size = new System.Drawing.Size(552, 440); this.axMediaPlayer1.TabIndex = 0; this.axMediaPlayer1.PlayStateChange += new AxMediaPlayer._MediaPlayerEvents_PlayStateChangeEventHandler(this.axMediaPlayer1_PlayStateChange); // // statusBar1 // this.statusBar1.Location = new System.Drawing.Point(0, 488); this.statusBar1.Name = "statusBar1"; this.statusBar1.Size = new System.Drawing.Size(552, 22); this.statusBar1.TabIndex = 1; // // inkPicture1 // this.inkPicture1.Anchor = (((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) | System.Windows.Forms.AnchorStyles.Left) | System.Windows.Forms.AnchorStyles.Right); this.inkPicture1.BackColor = System.Drawing.Color.Transparent; this.inkPicture1.BorderStyle = System.Windows.Forms.BorderStyle.Fixed3D; this.inkPicture1.Enabled = false; this.inkPicture1.Location = new System.Drawing.Point(0, 48); this.inkPicture1.MarginX = -2147483648; this.inkPicture1.MarginY = -2147483648; this.inkPicture1.Name = "inkPicture1"; this.inkPicture1.Size = new System.Drawing.Size(552, 360); this.inkPicture1.TabIndex = 2; this.inkPicture1.StrokesDeleting += new Microsoft.Ink.InkOverlayStrokesDeletingEventHandler(this.inkPicture1_StrokesDeleting); // // timer1 // this.timer1.Enabled = true; this.timer1.Tick += new System.EventHandler(this.timer1_Tick); // // colorDialog1 // this.colorDialog1.Color = System.Drawing.Color.Yellow; this.colorDialog1.FullOpen = true; this.colorDialog1.SolidColorOnly = true; // // clrBut // this.clrBut.Location = new System.Drawing.Point(256, 5); this.clrBut.Name = "clrBut"; this.clrBut.Size = new System.Drawing.Size(50, 40); this.clrBut.TabIndex = 3; this.clrBut.Text = "clear"; this.clrBut.Click += new System.EventHandler(this.clrBut_Click); // // colBut // this.colBut.Location = new System.Drawing.Point(64, 5); this.colBut.Name = "colBut"; this.colBut.Size = new System.Drawing.Size(50, 40); this.colBut.TabIndex = 4; this.colBut.Text = "color"; this.colBut.Click += new System.EventHandler(this.colBut_Click); // // penTrack // this.penTrack.LargeChange = 50; this.penTrack.Location = new System.Drawing.Point(120, 0); this.penTrack.Maximum = 200; this.penTrack.Minimum = 1; this.penTrack.Name = "penTrack"; this.penTrack.Size = new System.Drawing.Size(80, 45); this.penTrack.SmallChange = 50; this.penTrack.TabIndex = 5; this.penTrack.TickFrequency = 50; this.penTrack.TickStyle = System.Windows.Forms.TickStyle.Both; this.penTrack.Value = 50; this.penTrack.Scroll += new System.EventHandler(this.penTrack_Scroll); // // openFileDialog1 // this.openFileDialog1.AddExtension = false; this.openFileDialog1.Filter = "AVI|*.avi|MPG, MPEG|*.mpg;*.mpeg|ASF, WMV|*.ASF;*.WMV|ALL|*.*"; this.openFileDialog1.FilterIndex = 4; this.openFileDialog1.InitialDirectory = "c:\\only4gurus"; // // openBut // this.openBut.Location = new System.Drawing.Point(8, 5); this.openBut.Name = "openBut"; this.openBut.Size = new System.Drawing.Size(50, 40); this.openBut.TabIndex = 6; this.openBut.Text = "open"; this.openBut.Click += new System.EventHandler(this.openBut_Click); // // erBut // this.erBut.Location = new System.Drawing.Point(200, 5); this.erBut.Name = "erBut"; this.erBut.Size = new System.Drawing.Size(50, 40); this.erBut.TabIndex = 7; this.erBut.Text = "erase"; this.erBut.Click += new System.EventHandler(this.erBut_Click); // // gestBox // this.gestBox.BackColor = System.Drawing.Color.White; this.gestBox.ForeColor = System.Drawing.Color.Black; this.gestBox.Location = new System.Drawing.Point(488, 0); this.gestBox.Name = "gestBox"; this.gestBox.Size = new System.Drawing.Size(64, 48); this.gestBox.TabIndex = 8; this.gestBox.TabStop = false; this.gestBox.Text = "gesture"; // // ccBut // this.ccBut.Location = new System.Drawing.Point(424, 5); this.ccBut.Name = "ccBut"; this.ccBut.Size = new System.Drawing.Size(50, 40); this.ccBut.TabIndex = 9; this.ccBut.Text = "caption"; this.ccBut.Click += new System.EventHandler(this.ccBut_Click); // // spchBut // this.spchBut.Location = new System.Drawing.Point(312, 5); this.spchBut.Name = "spchBut"; this.spchBut.Size = new System.Drawing.Size(50, 40); this.spchBut.TabIndex = 10; this.spchBut.Text = "speech"; this.spchBut.Click += new System.EventHandler(this.spchBut_Click); // // ttsBut // this.ttsBut.Location = new System.Drawing.Point(368, 5); this.ttsBut.Name = "ttsBut"; this.ttsBut.Size = new System.Drawing.Size(50, 40); this.ttsBut.TabIndex = 11; this.ttsBut.Text = "ITS"; this.ttsBut.Click += new System.EventHandler(this.ttsBut_Click); // // TStrator // this.AutoScaleBaseSize = new System.Drawing.Size(5, 13); this.ClientSize = new System.Drawing.Size(552, 510); this.Controls.AddRange(new System.Windows.Forms.Control[] { this.ttsBut, this.spchBut, this.ccBut, this.gestBox, this.erBut, this.openBut, this.penTrack, this.colBut, this.clrBut, this.inkPicture1, this.statusBar1, this.axMediaPlayer1}); this.Name = "TStrator"; this.Text = "tabletStrator"; this.Resize += new System.EventHandler(this.Form1_Resize); this.Load += new System.EventHandler(this.Form1_Load); this.Move += new System.EventHandler(this.Form1_Move); this.Paint += new System.Windows.Forms.PaintEventHandler(this.Form1_Paint); ((System.ComponentModel.ISupportInitialize)(this.axMediaPlayer1)).EndInit(); ((System.ComponentModel.ISupportInitialize)(this.penTrack)).EndInit(); this.ResumeLayout(false); } #endregion /// /// The main entry point for the application. /// [STAThread] static void Main() { Application.Run(new TStrator()); } //inkPicture Graphics g; int oldWidth; int oldHeight; MyInkTag mit; //gesture private InkCollector ic; Recognizers r; private void Form1_Load(object sender, System.EventArgs e) { //Microsoft.Ink.Tablet t = inkPicture1.Tablet; //Microsoft.Ink.InkCollector ic; mit = new MyInkTag(); oldWidth = inkPicture1.Width; oldHeight = inkPicture1.Height; this.axMediaPlayer1.Size = new System.Drawing.Size(inkPicture1.Size.Width, inkPicture1.Size.Height+70); g = inkPicture1.CreateGraphics(); inkPicture1.DefaultDrawingAttributes.Color = Color.Yellow; inkPicture1.DefaultDrawingAttributes.AntiAliased = true; //gesture r = new Recognizers(); ic = new InkCollector(gestBox.Handle); ic.CollectionMode = CollectionMode.GestureOnly; ic.Gesture += new InkCollectorGestureEventHandler(this.GestureHandler); ic.SetGestureStatus(ApplicationGesture.AllGestures, false); ic.SetGestureStatus(ApplicationGesture.Square, true); ic.SetGestureStatus(ApplicationGesture.ChevronRight, true); ic.SetGestureStatus(ApplicationGesture.DownUp, true); ic.SetGestureStatus(ApplicationGesture.UpDown, true); ic.SetGestureStatus(ApplicationGesture.SemiCircleLeft, true); ic.SetGestureStatus(ApplicationGesture.SemiCircleRight, true); ic.Enabled = true; timer1.Enabled = true; } private void GestureHandler(object sender, InkCollectorGestureEventArgs e) { if (r.Count == 0) { statusBar1.Text = "gesture: no recognizers installed"; return; } foreach(Gesture g in e.Gestures) { if(g.Id == ApplicationGesture.NoGesture) continue; if(g.Confidence == RecognitionConfidence.Strong | g.Confidence == RecognitionConfidence.Intermediate) { if(g.Id == ApplicationGesture.Square) { statusBar1.Text = "gesture: square"; axMediaPlayer1.Stop(); } if(g.Id == ApplicationGesture.ChevronRight) { statusBar1.Text = "gesture: ChevronRight"; axMediaPlayer1.Play(); } if(g.Id == ApplicationGesture.UpDown | g.Id == ApplicationGesture.DownUp) { statusBar1.Text = "gesture: UpDown OR DownUp"; axMediaPlayer1.Pause(); } if(g.Id == ApplicationGesture.SemiCircleLeft) { statusBar1.Text = "gesture: SemiCircleLeft"; axMediaPlayer1.Mute = true; } if(g.Id == ApplicationGesture.SemiCircleRight) { statusBar1.Text = "gesture: SemiCircleRight"; axMediaPlayer1.Mute = false; } } else { statusBar1.Text = "gesture: low confidence"; } } } private void axMediaPlayer1_PlayStateChange(object sender, AxMediaPlayer._MediaPlayerEvents_PlayStateChangeEvent e) { if(e.newState == (int) MediaPlayer.MPPlayStateConstants.mpPlaying) { inkPicture1.Enabled = true; this.RedrawInk(RedrawEnum.Play); } else if(e.newState == (int) MediaPlayer.MPPlayStateConstants.mpPaused) { inkPicture1.Enabled = true; this.RedrawInk(RedrawEnum.Pause); } else inkPicture1.Enabled = false; } public void Scale() { try { int newWidth = inkPicture1.Width; int newHeight = inkPicture1.Height; if(newWidth!=0 & newHeight!=0 & inkPicture1.Ink.Strokes.Count>0) { float wScale = (float)newWidth / (float)oldWidth; float hScale = (float)newHeight / (float)oldHeight; Matrix m = new Matrix(); m.Scale(wScale, hScale); //m.Translate(-hScrollBar.Value, -vScrollBar.Value); //view modifies pen width too, object does not inkPicture1.Renderer.SetViewTransform(m); //myInkCollector.Renderer.SetObjectTransform(m); //statusBar1.Text = "newW: " + newWidth.ToString() + ", newH: " + newHeight.ToString() + ", wScale: " + wScale.ToString() + ", hScale: " + hScale.ToString(); //oldWidth = newWidth; //oldHeight = newHeight; } } catch(Exception ex) { MessageBox.Show(ex.ToString()); } } //TODO it misses redraw on this one too often? //it is worse when it is paused and then moved private void Form1_Move(object sender, System.EventArgs e) { this.RedrawInk(RedrawEnum.Move); } private void Form1_Resize(object sender, System.EventArgs e) { Scale(); this.RedrawInk(RedrawEnum.Resize); } private void timer1_Tick(object sender, System.EventArgs e) { timer1.Stop(); if(mit.Redraw==true & inkPicture1.Ink.Strokes.Count>0) { //this does not draw new strokes, just existing one(s) if(mit.RedrawType == RedrawEnum.Move) { mit.RedrawCount = mit.RedrawCount + 1; if(mit.RedrawCount == 13) //lucky magic #13 { mit.Redraw = false; mit.RedrawCount = 0; } } else { mit.Redraw = false; } inkPicture1.Renderer.Draw(g, inkPicture1.Ink.Strokes); //statusBar1.Text = "redraw: " + DateTime.Now.ToLongTimeString(); } timer1.Start(); } private void RedrawInk(RedrawEnum re) { if(mit != null) { mit.RedrawType = re; mit.Redraw = true; } } private void Form1_Paint(object sender, System.Windows.Forms.PaintEventArgs e) { this.RedrawInk(RedrawEnum.Paint); } private void clrBut_Click(object sender, System.EventArgs e) { inkPicture1.Ink.DeleteStrokes(); axMediaPlayer1.Invalidate(inkPicture1.Ink.GetBoundingBox()); //axMediaPlayer1.Refresh(); } private void colBut_Click(object sender, System.EventArgs e) { DialogResult dr = colorDialog1.ShowDialog(this); if(DialogResult.OK == dr) { inkPicture1.DefaultDrawingAttributes.Color = colorDialog1.Color; //this.RedrawInk(RedrawEnum.Color); } //axMediaPlayer1.Refresh(); this.Refresh(); } private void penTrack_Scroll(object sender, System.EventArgs e) { float width = penTrack.Value; inkPicture1.DefaultDrawingAttributes.Width = (float) penTrack.Value; } private void openBut_Click(object sender, System.EventArgs e) { DialogResult dr = openFileDialog1.ShowDialog(this); if(DialogResult.OK == dr) { this.clrBut_Click(null, null); axMediaPlayer1.FileName = openFileDialog1.FileName; } this.Refresh(); } private void erBut_Click(object sender, System.EventArgs e) { if(erBut.Text == "erase") { erBut.Text = "ink"; inkPicture1.EditingMode = InkOverlayEditingMode.Delete; } else { erBut.Text = "erase"; inkPicture1.EditingMode = InkOverlayEditingMode.Ink; } } private void inkPicture1_StrokesDeleting(object sender, Microsoft.Ink.InkOverlayStrokesDeletingEventArgs e) { inkPicture1.Ink.DeleteStrokes(e.StrokesToDelete); axMediaPlayer1.Invalidate(inkPicture1.Ink.GetBoundingBox()); axMediaPlayer1.Refresh(); this.RedrawInk(RedrawEnum.Erase); } //tried to make it do dictation against video stream as auto closed captioning private void ccBut_Click(object sender, System.EventArgs e) { SpeechLib.SpSharedRecoContextClass ssrc = null; SpeechLib.ISpeechRecoGrammar isrg = null; ssrc = new SpeechLib.SpSharedRecoContextClass(); ssrc.Recognition += new SpeechLib._ISpeechRecoContextEvents_RecognitionEventHandler(RecognitionEvent); isrg = ssrc.CreateGrammar(1); isrg.DictationLoad(null, SpeechLib.SpeechLoadOption.SLOStatic); isrg.DictationSetState(SpeechLib.SpeechRuleState.SGDSActive); } public void RecognitionEvent(int i, object o, SpeechLib.SpeechRecognitionType srt, SpeechLib.ISpeechRecoResult isrr) { string strText = isrr.PhraseInfo.GetText(0, -1, true); //what the h3ll statusBar1.Text = "recognized: " + strText; } private void ttsBut_Click(object sender, System.EventArgs e) { TtsForm tf = new TtsForm(); tf.Show(); } SpSharedRecoContext ssrc; ISpeechRecoGrammar isrg; private void spchBut_Click(object sender, System.EventArgs e) { ssrc = new SpSharedRecoContext(); object grammarId = 101; //0 is default isrg = ssrc.CreateGrammar(grammarId); string gramFile = @"C:\_DEV\tabletStrator\gram.xml"; isrg.CmdLoadFromFile(gramFile, SpeechLoadOption.SLODynamic); isrg.CmdSetRuleIdState(0, SpeechRuleState.SGDSActive); ssrc.FalseRecognition += new SpeechLib._ISpeechRecoContextEvents_FalseRecognitionEventHandler(ssrc_FalseRecognition); ssrc.Recognition += new SpeechLib._ISpeechRecoContextEvents_RecognitionEventHandler(ssrc_Recognition); ssrc.StartStream += new SpeechLib._ISpeechRecoContextEvents_StartStreamEventHandler(ssrc_StartStream); statusBar1.Text = "command and control is ready"; } private void ssrc_FalseRecognition(int StreamNumber, object StreamPosition, ISpeechRecoResult Result) { statusBar1.Text = "False Recognition"; } private void ssrc_Recognition(int StreamNumber, object StreamPosition, SpeechRecognitionType RecognitionType, ISpeechRecoResult Result) { string res = String.Empty; foreach(SpeechLib.ISpeechPhraseElement ispe in Result.PhraseInfo.Elements) { res += ispe.DisplayText; this.RecoAction(res); } statusBar1.Text = "Recognition: " + res + " "; //statusBar1.Text = "Recognition: "; } private void RecoAction(string command) { int val = 0; switch(command) { case "open": this.openBut_Click(null, null); break; case "color": this.colBut_Click(null, null); break; case "thinner": val = this.penTrack.Value; this.penTrack.Value = val - 40; this.penTrack_Scroll(null, null); break; case "thicker": val = this.penTrack.Value; this.penTrack.Value = val + 40; this.penTrack_Scroll(null, null); break; case "ink": this.erBut_Click(null, null); break; case "erase": this.erBut_Click(null, null); break; case "clear": this.clrBut_Click(null, null); break; case "play": axMediaPlayer1.Play(); break; case "pause": axMediaPlayer1.Pause(); break; case "stop": axMediaPlayer1.Stop(); break; case "mute": axMediaPlayer1.Mute = true; break; case "unmute": axMediaPlayer1.Mute = false; break; case "help": Help h = new Help(); h.ShowDialog(); this.Refresh(); break; default: //should not reach here break; } } private void ssrc_StartStream(int StreamNumber, object StreamPosition) { statusBar1.Text = "StartStream: " + StreamNumber.ToString(); } /* Public WithEvents RC As SpSharedRecoContext Public myGrammar, b As ISpeechRecoGrammar Private Sub Form_Load() Set RC = New SpSharedRecoContext Set myGrammar = RC.CreateGrammar myGrammar.CmdLoadFromFile "sol.xml", SLODynamic myGrammar.CmdSetRuleIdState 0, SGDSActive End Sub Private Sub RC_FalseRecognition(ByVal StreamNumber As Long, ByVal StreamPosition As Variant, ByVal Result As SpeechLib.ISpeechRecoResult) 'Label1.Caption = "(no recognition)" End Sub Private Sub RC_Recognition(ByVal StreamNumber As Long, ByVal StreamPosition As Variant, ByVal RecognitionType As SpeechLib.SpeechRecognitionType, ByVal Result As SpeechLib.ISpeechRecoResult) 'Label1.Caption = Result.PhraseInfo.GetText End Sub Private Sub RC_StartStream(ByVal StreamNumber As Long, ByVal StreamPosition As Variant) 'Label2.Caption = Val(StreamNumber) End Sub */ } }