How can I get two threads to run simultaneously in a single process?

Experimenting with MS Text to Speech (TTS) I built a simple dialog that speaks a greeting when a button is pressed. I wish to incorporate an avatar that mouths the spoken words. Without getting into all the details of how to match the phenomes with the facial movements, I just included an avatar that moves it's mouth.

I cannot figure out how to coordinate the speech and the avatar movement. My first attempt was to try to run them both together, but that doesnt work -- whichever one runs first must complete before the other runs. My second attempt was to interleave a spoken word with a change in bitmap image -- that sort of works but is very slow and jerky. (see attached app). Perhaps a better approach would be to use multithreading and run speech and avatar moves each as separate worker threads. Sounds good, but I do not know how to do that. And after spending many hours reading books and scouring the web for information on Win API and MFC multithreading, I am more confused than ever.

I would appreciate your thoughts on the problem and how you might go about trying to solve it. Thanks.

You can get your result in a single thread, by enabling the eventing model in the TTS api.

See "Using Events with TTS (SAPI 5.3)" at http://msdn.microsoft.com/en-us/libr...20165(v=vs.85)

The event that you want is probably the viseme event, which (per the documentation) can be used to animate a face.

Mike

Exactly what I was looking for. Thanks, Mike.

Thanks to MikeAThon I was able to adapt code from the following site:
http://msdn.microsoft.com/en-us/libr...(v=VS.85).aspx
With slight modification, I was able to trap the visemes.

Code:

---

// InitInstance
//..
  //Initialize SAPI
  HRESULT hr = CoCreateInstance(CLSID_SpVoice, NULL, CLSCTX_ALL, IID_ISpVoice, (void **)&pVoice);
  if( SUCCEEDED( hr ) )
  {
      pVoice->SetInterest( SPFEI(SPEI_VISEME),SPFEI(SPEI_VISEME) );
      pVoice->SetNotifyWindowMessage( hWnd, WM_USER, 0, 0 );
  }
      else
      return FALSE;
//..
// In WndProc

      case WM_USER:
        SPEVENT eventItem;
        memset( &eventItem, 0,sizeof(SPEVENT));
        while( pVoice->GetEvents(1, &eventItem, NULL ) == S_OK )
        {
          switch(eventItem.eEventId )
          {
          case SPEI_VISEME :
                pVoice->GetStatus( &eventStatus, NULL );
                                ULONG viseme;
                                viseme = eventStatus.VisemeId;

                                _RPT1(0, "viseme = %d\n", viseme);
                                  break;

              default:
                break;
          }

        SpClearEvent( &eventItem );
        }
        break;

---

I was also able to load a bitmap image ("mouth6.bmp") and display it on the window using

Code:

---

        case WM_CREATE:       
                hBitmap = (HBITMAP)LoadImage(hInst, L"mouth4.bmp", IMAGE_BITMAP, 0, 0, LR_LOADFROMFILE);       
                break;   
        case WM_PAINT:       
                PAINTSTRUCT    ps;       
                HDC                    hdc;       
                BITMAP                  bitmap;       
                HDC                    hdcMem;       
                HGDIOBJ                oldBitmap;         
                hdc = BeginPaint(hWnd, &ps);         
                hdcMem = CreateCompatibleDC(hdc);       
                oldBitmap = SelectObject(hdcMem, hBitmap);         
                GetObject(hBitmap, sizeof(bitmap), &bitmap);       
                BitBlt(hdc, 0, 0, bitmap.bmWidth, bitmap.bmHeight, hdcMem, 0, 0, SRCCOPY);         
                SelectObject(hdcMem, oldBitmap);       
                DeleteDC(hdcMem);         
                EndPaint(hWnd, &ps);       
                break;   
        case WM_DESTROY:       
                DeleteObject(hBitmap);       
                PostQuitMessage(0);       
                break;

---

But the problem remains of how to coordinate the bitmap viseme images with the speech visemes, assuming that we have a collection of viseme bitmap images that correspond to the 21 or so possible viseme events. Any idea how this might be accomplished ?

It's my understanding that the TTS SDK includes a sample program that does exactly what you are attempting, using a "talking microphone" with different mouth shapes for each different viseme. Look in the SDK for files like mic_eyes_narrow.bmp and mic_mouth_2.bmp

If you can't find it, then it seems that this article explains the SDK sample code: "Converting Text-To-Speech and and using mouth motion animation" by Agus Kurniawan at http://www.codeproject.com/Articles/...ing-mouth-moti

Quote:

---

Originally Posted by Mike Pliam

Any idea how this might be accomplished ?

---

See your modified sample attached.

Code:

---

                          case SPEI_VISEME :
                        // Viseme was determined by synthesis engine. The high word of wParam is the duration, in milliseconds,
                        // of the current viseme. The low word is for the next viseme of type SPVISEMES. The high word of lParam
                        // is the viseme feature defined in SPVFEATURE. This value will be zero if the current viseme is not primary
                        // stress or emphasis. The low word of lParam is the current viseme being synthesized.
                        // wmId    = LOWORD(wParam);
                        // wmEvent = HIWORD(wParam);

                                //_RPT1(0, "wmId = %d\n", wmId);
                                //_RPT1(0, "wmEvent = %d\n", wmEvent);

                pVoice->GetStatus( &eventStatus, NULL );
                                ULONG viseme;
                                viseme = eventStatus.VisemeId;

                                _RPT1(0, "viseme = %d\n", viseme);
                                // now you know what viseme to render
                                // animation goes here below
                                // decide what bitmap to draw
                                bmpIdx = (viseme > 11); // simple animation:
                                                        // consonants are above 11,
                                                        // vowels are up to, therefore,
                                                        // mouth4.bmp will go for vowels,
                                                        // and test.bmp for consonants

                                // make your window to re-paint the sprite region
                                InvalidateRect(hWnd, NULL /* in fact you need your sprite rect here*/, FALSE);
                                  break;

              default:
                break;
          }

        SpClearEvent( &eventItem );
        }
        break;

        case WM_CREATE:       
                // pre-load sprite images
                hBitmap[0] = (HBITMAP)LoadImage(hInst, L"mouth4.bmp", IMAGE_BITMAP, 0, 0, LR_LOADFROMFILE);       
                hBitmap[1] = (HBITMAP)LoadImage(hInst, L"test.bmp", IMAGE_BITMAP, 0, 0, LR_LOADFROMFILE);       
                break;   
        case WM_PAINT:       
                PAINTSTRUCT    ps;       
                HDC                    hdc;       
                BITMAP                  bitmap;       
                HDC                    hdcMem;       
                HGDIOBJ                oldBitmap;         
                hdc = BeginPaint(hWnd, &ps);         
                hdcMem = CreateCompatibleDC(hdc);       
                oldBitmap = SelectObject(hdcMem, hBitmap[bmpIdx]);         
                GetObject(hBitmap[bmpIdx], sizeof(bitmap), &bitmap);       
                BitBlt(hdc, 0, 0, bitmap.bmWidth, bitmap.bmHeight, hdcMem, 0, 0, SRCCOPY);         
                SelectObject(hdcMem, oldBitmap);       
                DeleteDC(hdcMem);         
                EndPaint(hWnd, &ps);       
                break;

---

Thankyou very much for taking time to rework my code, Igor. Your methods work nicely. :)

I bet it does.

Ideally, you provide a mouth shape sprite for every viseme from the range, and use the viseme value as an index to the array of sprites.