Skip to content

Commit b2fc36f

Browse files
compulimcompulim
authored
Added speech synthesis bypass engine (#2445)
* Added no-op speech synthesis engine * Update PR number * Update comment * Fix ESLint and rename to bypass * Update CHANGELOG.md Co-Authored-By: TJ Durnford <[email protected]> * Apply PR comments
1 parent 903400a commit b2fc36f

5 files changed

Lines changed: 1064 additions & 5 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
9696
- Fix [#2360](https://github.com/microsoft/BotFramework-WebChat/issues/2360). Timestamp should update on language change, by [@compulim](https://github.com/compulim) in PR [#2414](https://github.com/microsoft/BotFramework-WebChat/pull/2414)
9797
- Fix [#2428](https://github.com/microsoft/BotFramework-WebChat/issues/2428). Should interrupt speech synthesis after microphone button is clicked, by [@compulim](https://github.com/compulim) in PR [#2429](https://github.com/microsoft/BotFramework-WebChat/pull/2429)
9898
- Fix [#2422](https://github.com/microsoft/BotFramework-WebChat/issues/2422). Store thumbnail URL using the activity's `attachment.thumbnailUrl` field, by [@compulim](https://github.com/compulim) in PR [#2433](https://github.com/microsoft/BotFramework-WebChat/pull/2433)
99+
- Fix [#2435](https://github.com/microsoft/BotFramework-WebChat/issues/2435). Fix microphone button getting stuck on voice-triggered expecting input hint without a speech synthesis engine, by [@compulim](https://github.com/compulim) in PR [#2445](https://github.com/microsoft/BotFramework-WebChat/pull/2445)
99100

100101
### Added
101102

‎__tests__/speech.synthesis.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,4 +124,28 @@ describe('speech synthesis', () => {
124124
await expect(speechRecognitionStartCalled().fn(driver)).resolves.toBeTruthy();
125125
await driver.wait(negateCondition(speechSynthesisUtterancePended()), timeouts.ui);
126126
});
127+
128+
describe('without speech synthesis', () => {
129+
test('should start recognition immediately after receiving expected input hint', async () => {
130+
const { driver, pageObjects } = await setupWebDriver({
131+
props: {
132+
webSpeechPonyfillFactory: () => {
133+
const { SpeechGrammarList, SpeechRecognition } = window.WebSpeechMock;
134+
135+
return {
136+
SpeechGrammarList,
137+
SpeechRecognition
138+
};
139+
}
140+
}
141+
});
142+
143+
await pageObjects.sendMessageViaMicrophone('input hint expected');
144+
145+
await driver.wait(minNumActivitiesShown(2), timeouts.directLine);
146+
147+
await expect(speechRecognitionStartCalled().fn(driver)).resolves.toBeTruthy();
148+
await driver.wait(negateCondition(speechSynthesisUtterancePended()), timeouts.ui);
149+
});
150+
});
127151
});

‎packages/component/src/BasicTranscript.js‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ import connectToWebChat from './connectToWebChat';
99
import ScrollToEndButton from './Activity/ScrollToEndButton';
1010
import SpeakActivity from './Activity/Speak';
1111

12+
import {
13+
speechSynthesis as bypassSpeechSynthesis,
14+
SpeechSynthesisUtterance as BypassSpeechSynthesisUtterance
15+
} from './Speech/BypassSpeechSynthesisPonyfill';
16+
1217
const ROOT_CSS = css({
1318
overflow: 'hidden',
1419
position: 'relative'
@@ -85,7 +90,11 @@ const BasicTranscript = ({
8590
<div className={classNames(ROOT_CSS + '', className + '')} role="log">
8691
<ScrollToBottomPanel className={PANEL_CSS + ''}>
8792
<div className={FILLER_CSS} />
88-
<SayComposer speechSynthesis={speechSynthesis} speechSynthesisUtterance={SpeechSynthesisUtterance}>
93+
<SayComposer
94+
// These are props for passing in Web Speech ponyfill, where speech synthesis requires these two class/object to be ponyfilled.
95+
speechSynthesis={speechSynthesis || bypassSpeechSynthesis}
96+
speechSynthesisUtterance={SpeechSynthesisUtterance || BypassSpeechSynthesisUtterance}
97+
>
8998
<ul
9099
aria-atomic="false"
91100
aria-live="polite"
@@ -95,7 +104,7 @@ const BasicTranscript = ({
95104
>
96105
{activityElements.map(({ activity, element }, index) => (
97106
<li
98-
/* Because of differences in browser implementations, aria-label=" " is used to make the screen reader not repeat the same text multiple times in Chrome v75 */
107+
// Because of differences in browser implementations, aria-label=" " is used to make the screen reader not repeat the same text multiple times in Chrome v75
99108
aria-label=" "
100109
className={classNames(styleSet.activity + '', {
101110
// Hide timestamp if same timestamp group with the next activity
@@ -110,9 +119,7 @@ const BasicTranscript = ({
110119
>
111120
{element}
112121
{// TODO: [P2] We should use core/definitions/speakingActivity for this predicate instead
113-
speechSynthesis && activity.channelData && activity.channelData.speak && (
114-
<SpeakActivity activity={activity} />
115-
)}
122+
activity.channelData && activity.channelData.speak && <SpeakActivity activity={activity} />}
116123
</li>
117124
))}
118125
</ul>

‎packages/component/src/Speech/BypassSpeechSynthesisPonyfill.js‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Since this is a bypass, we will relax some ESLint rules.
2+
// All classes/properties defined here are in W3C Web Speech API.
3+
4+
/* eslint class-methods-use-this: "off" */
5+
/* eslint getter-return: "off" */
6+
/* eslint max-classes-per-file: ["error", 4] */
7+
/* eslint no-empty-function: "off" */
8+
9+
import EventTarget, { defineEventAttribute } from '../external/event-target-shim';
10+
11+
class SpeechSynthesisEvent {
12+
constructor(type, utterance) {
13+
this._type = type;
14+
this._utterance = utterance;
15+
}
16+
17+
get charIndex() {
18+
return 0;
19+
}
20+
21+
get elapsedTime() {
22+
return 0;
23+
}
24+
25+
get name() {}
26+
27+
get type() {
28+
return this._type;
29+
}
30+
31+
get utterance() {
32+
return this._utterance;
33+
}
34+
}
35+
36+
class SpeechSynthesisUtterance extends EventTarget {
37+
constructor(text) {
38+
super();
39+
40+
this._lang = 'en-US';
41+
this._pitch = 1;
42+
this._rate = 1;
43+
this._text = text;
44+
this._voice = null;
45+
this._volume = 1;
46+
}
47+
48+
get lang() {
49+
return this._lang;
50+
}
51+
52+
set lang(value) {
53+
this._lang = value;
54+
}
55+
56+
get pitch() {
57+
return this._pitch;
58+
}
59+
60+
set pitch(value) {
61+
this._pitch = value;
62+
}
63+
64+
get rate() {
65+
return this._rate;
66+
}
67+
68+
set rate(value) {
69+
this._rate = value;
70+
}
71+
72+
get text() {
73+
return this._text;
74+
}
75+
76+
set text(value) {
77+
this._text = value;
78+
}
79+
80+
get voice() {
81+
return this._voice;
82+
}
83+
84+
set voice(value) {
85+
this._voice = value;
86+
}
87+
88+
get volume() {
89+
return this._volume;
90+
}
91+
92+
set volume(value) {
93+
this._volume = value;
94+
}
95+
}
96+
97+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'boundary');
98+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'end');
99+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'error');
100+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'mark');
101+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'pause');
102+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'resume');
103+
defineEventAttribute(SpeechSynthesisUtterance.prototype, 'start');
104+
105+
class SpeechSynthesisVoice {
106+
get default() {
107+
return true;
108+
}
109+
110+
get lang() {
111+
return 'en-US';
112+
}
113+
114+
get localService() {
115+
return true;
116+
}
117+
118+
get name() {
119+
return 'English (US)';
120+
}
121+
122+
get voiceURI() {
123+
return 'English (US)';
124+
}
125+
}
126+
127+
class SpeechSynthesis extends EventTarget {
128+
get paused() {
129+
return false;
130+
}
131+
132+
get pending() {
133+
return false;
134+
}
135+
136+
get speaking() {
137+
return false;
138+
}
139+
140+
cancel() {}
141+
142+
getVoices() {
143+
return [new SpeechSynthesisVoice()];
144+
}
145+
146+
pause() {
147+
throw new Error('pause is not implemented.');
148+
}
149+
150+
resume() {
151+
throw new Error('resume is not implemented.');
152+
}
153+
154+
speak(utterance) {
155+
utterance.dispatchEvent(new SpeechSynthesisEvent('start', utterance));
156+
utterance.dispatchEvent(new SpeechSynthesisEvent('end', utterance));
157+
}
158+
}
159+
160+
defineEventAttribute(SpeechSynthesis.prototype, 'voiceschanged');
161+
162+
const speechSynthesis = new SpeechSynthesis();
163+
164+
export { speechSynthesis, SpeechSynthesisEvent, SpeechSynthesisUtterance, SpeechSynthesisVoice };

0 commit comments

Comments
 (0)