diff --git a/examples/lex/chatpickle/schedule-appointment-bot.feature b/examples/lex/chatpickle/schedule-appointment-bot.feature index ebee194..f850cf0 100644 --- a/examples/lex/chatpickle/schedule-appointment-bot.feature +++ b/examples/lex/chatpickle/schedule-appointment-bot.feature @@ -10,4 +10,6 @@ Feature: ScheduleAppointment Bot * Bot: /At what time/ * User: four pm * Bot: /\d{2}:\d{2} is available, should I go ahead and book your appointment\?/ - * User: yes \ No newline at end of file + * User: yes + + diff --git a/examples/lexVoice/README_LEX.md b/examples/lexVoice/README_LEX.md new file mode 100644 index 0000000..347ec73 --- /dev/null +++ b/examples/lexVoice/README_LEX.md @@ -0,0 +1,23 @@ +# AWS Lex Example + +This sample project, [examples/lex](./) and [examples/lexVoice](./), is designed to be pointed at the OrderFlowers and ScheduleAppointment bots as provided by AWS as a blueprint. +[examples/lex](./) uses the lex postText to post user responses and [examples/lexVoice](./) uses the lex postContent. + +Using the lexVoice, we can post audio content as the user response. Please take a look at [examples/lexVoice/chatpickle/schedule-appointment-bot.feature](./) where the audio file name is listed as the user response. Only pcm files are supported. We can also add mp3 and mpeg support in the future if needed. + +### Create a config file + +You need a [chatpickle.config.json](chatpickle.config.json) (or .js) in the root of your node.js project and it should be formatted like the example provided. + +### Create a chatpickle/ folder +You also need a [chatpickle/](chatpickle) folder in the root of your project. This is where you will put your gherkin feature files which can leverage the extended chatpickle syntax. + +### Setup a Lex Chatbot +To setup your own OrderFlowers and ScheduleAppointment bots from a blueprint in your AWS account, follow this [AWS Lex Guide](https://docs.aws.amazon.com/lex/latest/dg/gs-bp-create-bot.html). + +Or if you want to get started with chatpickling a custom lex bot, alter your project's chatpickle config; see example at [chatpickle.config.json](chatpickle.config.json) + +### Setup IAM Credentials +You will need to create IAM credentials that can invoke your bot. You can use the Amazon managed policies as shown below. Load them in your `.aws` directory or follow this [AWS CLI Configure Guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). + +![Lex Execution IAM Credentials](https://miro.medium.com/max/750/0*m55m6A95OcpcFRDa.png) diff --git a/examples/lexVoice/chatpickle.config.json b/examples/lexVoice/chatpickle.config.json new file mode 100644 index 0000000..0f3f75c --- /dev/null +++ b/examples/lexVoice/chatpickle.config.json @@ -0,0 +1,33 @@ +{ + "bots": { + "OrderFlowers": { + "type": "LexVoice", + "context": { + "botName": "OrderFlowers", + "botAlias": "prod", + "region": "us-east-1" + } + }, + "ScheduleAppointment": { + "type": "LexVoice", + "context": { + "botName": "ScheduleAppointment", + "botAlias": "prod", + "region": "us-east-1" + } + } + }, + "users": { + "homer": { + "description": "Basic User Profile", + "context": { + "userId": "homer", + "userAttributes": { + "firstName": "Homer", + "lastName": "Simpson", + "address": "Springfield" + } + } + } + } +} \ No newline at end of file diff --git a/examples/lexVoice/chatpickle/UserAudioResponses/cleaning.pcm b/examples/lexVoice/chatpickle/UserAudioResponses/cleaning.pcm new file mode 100644 index 0000000..75b07ce Binary files /dev/null and b/examples/lexVoice/chatpickle/UserAudioResponses/cleaning.pcm differ diff --git a/examples/lexVoice/chatpickle/UserAudioResponses/fourPm.pcm b/examples/lexVoice/chatpickle/UserAudioResponses/fourPm.pcm new file mode 100644 index 0000000..e8d1df4 Binary files /dev/null and b/examples/lexVoice/chatpickle/UserAudioResponses/fourPm.pcm differ diff --git a/examples/lexVoice/chatpickle/UserAudioResponses/likeToBookAppointment.pcm b/examples/lexVoice/chatpickle/UserAudioResponses/likeToBookAppointment.pcm new file mode 100644 index 0000000..6cd4df4 Binary files /dev/null and b/examples/lexVoice/chatpickle/UserAudioResponses/likeToBookAppointment.pcm differ diff --git a/examples/lexVoice/chatpickle/UserAudioResponses/test.txt b/examples/lexVoice/chatpickle/UserAudioResponses/test.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/lexVoice/chatpickle/UserAudioResponses/tomorrow.pcm b/examples/lexVoice/chatpickle/UserAudioResponses/tomorrow.pcm new file mode 100644 index 0000000..b487301 Binary files /dev/null and b/examples/lexVoice/chatpickle/UserAudioResponses/tomorrow.pcm differ diff --git a/examples/lexVoice/chatpickle/UserAudioResponses/yes.pcm b/examples/lexVoice/chatpickle/UserAudioResponses/yes.pcm new file mode 100644 index 0000000..97302f2 Binary files /dev/null and b/examples/lexVoice/chatpickle/UserAudioResponses/yes.pcm differ diff --git a/examples/lexVoice/chatpickle/order-flowers-bot.feature b/examples/lexVoice/chatpickle/order-flowers-bot.feature new file mode 100644 index 0000000..ae3b4ad --- /dev/null +++ b/examples/lexVoice/chatpickle/order-flowers-bot.feature @@ -0,0 +1,27 @@ +Feature: OrderFlowers Bot + + Scenario: Anonymous orders roses for tomorrow at 4pm + Given the user begins a new chat with "OrderFlowers" + * User: I would like to order some flowers + * Bot: What type of flowers would you like to order? + * User: roses + * Bot: What day do you want the roses to be picked up? + * User: tomorrow + * Bot: /^Pick up the roses at what time on \d{4}-\d{2}-\d{2}\?$/ + * User: four pm + * Bot: /^Okay, your roses will be ready for pickup by 16:00 on \d{4}-\d{2}-\d{2}. Does this sound okay\?$/ + Then slots.FlowerType = roses + And sessionAttributes.FlowerType = undefined + + Scenario: Anonymous orders roses for tomorrow at 5pm but decides to cancel + Given the user begins a new chat with "OrderFlowers" + * User: I would like to order some flowers + * Bot: What type of flowers would you like to order? + * User: roses + * Bot: What day do you want the roses to be picked up? + * User: tomorrow + * Bot: /^Pick up the roses at what time on \d{4}-\d{2}-\d{2}\?$/ + * User: five pm + * Bot: /^Okay, your roses will be ready for pickup by 17:00 on \d{4}-\d{2}-\d{2}. Does this sound okay\?$/ + * User: no + * Bot: Okay, I will not place your order. \ No newline at end of file diff --git a/examples/lexVoice/chatpickle/schedule-appointment-bot.feature b/examples/lexVoice/chatpickle/schedule-appointment-bot.feature new file mode 100644 index 0000000..470bca0 --- /dev/null +++ b/examples/lexVoice/chatpickle/schedule-appointment-bot.feature @@ -0,0 +1,44 @@ +Feature: ScheduleAppointment Bot + + Scenario: Happy Path - Anonymous sets up cleaning appointment + Given the user begins a new chat with "ScheduleAppointment" + * User: I would like to book an appointment + * Bot: What type of appointment would you like to schedule? + * User: cleaning + * Bot: When should I schedule your cleaning? + * User: tomorrow + * Bot: /At what time/ + * User: four pm + * Bot: /\d{2}:\d{2} is available, should I go ahead and book your appointment\?/ + * User: yes + + Scenario: Non-Happy Path - Caller stays quiet. Bot should repeat the last prompt + Given the user begins a new chat with "ScheduleAppointment" + * User: I would like to book an appointment + * Bot: What type of appointment would you like to schedule? + * User: + * Bot: What type of appointment would you like to schedule? + * User: cleaning + * Bot: When should I schedule your cleaning? + * User: + * Bot: When should I schedule your cleaning? + * User: tomorrow + * Bot: /At what time/ + * User: + * Bot: /At what time/ + * User: four pm + * Bot: /\d{2}:\d{2} is available, should I go ahead and book your appointment\?/ + * User: yes + + Scenario: Use File + Given the user begins a new chat with "ScheduleAppointment" + * User: likeToBookAppointment.pcm + * Bot: What type of appointment would you like to schedule? + * User: cleaning.pcm + * Bot: When should I schedule your cleaning? + * User: tomorrow.pcm + * Bot: /At what time/ + * User: fourPm.pcm + * Bot: /\d{2}:\d{2} is available, should I go ahead and book your appointment\?/ + * User: yes.pcm + diff --git a/package.json b/package.json index 7224720..a652976 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "build:other": "cp -r src/cucumberSupport dist/cucumberSupport", "start": "npm run example:lex", "example:lex": "npm run build && node dist/cli.js --cpPath examples/lex/", + "example:lexVoice": "npm run build && node dist/cli.js --cpPath examples/lexVoice/", "example:custom": "npm run build && node dist/cli.js --cpPath examples/custom/", "test": "jest --reporters=default --reporters=jest-junit", "lint": "eslint \"{scripts,src,test}/**/*.{js,ts}\"", diff --git a/src/lib/botClients/LexClient.ts b/src/lib/botClients/LexClient.ts index b49c66f..fb2addb 100644 --- a/src/lib/botClients/LexClient.ts +++ b/src/lib/botClients/LexClient.ts @@ -44,7 +44,7 @@ export default class LexClient extends BotClient { this.lastResponse = await this.lex.postText(params).promise(); this.sessionAttributes = this.lastResponse.sessionAttributes; - const reply: string = this.lastResponse.message.trim(); + const reply: string = this.lastResponse.message? this.lastResponse.message.trim(): null; console.log(`[${this.userId}] Bot: ${reply}`); diff --git a/src/lib/botClients/LexVoiceClient.test.ts b/src/lib/botClients/LexVoiceClient.test.ts new file mode 100644 index 0000000..59bd3df --- /dev/null +++ b/src/lib/botClients/LexVoiceClient.test.ts @@ -0,0 +1,36 @@ +// Need to bypass type safety of typescript to allow this approach for mocking to work. +// eslint-disable-next-line @typescript-eslint/no-var-requires +const LexRuntime = require('aws-sdk/clients/lexruntime'); +import LexVoiceClient from './LexVoiceClient'; + +jest.mock('aws-sdk/clients/lexruntime'); + +const lexRuntimePostTextPromise = jest.fn().mockReturnValue({ + promise: jest.fn().mockResolvedValue({ + sessionAttributes: { foo: 'bar' }, + message: 'This is a mocked message.', + }), +}); + +LexRuntime.mockImplementation(() => ({ + postContent: lexRuntimePostTextPromise, +})); + +test('LexVoiceClient.speak()', async (): Promise => { + const botContext = { + botName: 'OrderFlowers', + botAlias: 'prod', + region: 'us-east-1', + }; + const userContext = { + userId: 'homer', + userAttributes: { + firstName: 'Homer', + lastName: 'Simpson', + address: 'Springfield', + }, + }; + const botClient = new LexVoiceClient(botContext, userContext); + const reply = await botClient.speak('Hello World'); + expect(reply).toBe('This is a mocked message.'); +}); diff --git a/src/lib/botClients/LexVoiceClient.ts b/src/lib/botClients/LexVoiceClient.ts new file mode 100644 index 0000000..37b2402 --- /dev/null +++ b/src/lib/botClients/LexVoiceClient.ts @@ -0,0 +1,120 @@ +import LexRuntime from 'aws-sdk/clients/lexruntime'; +import get from 'lodash.get'; +import { BotClient } from './BotClient'; +import Polly from 'aws-sdk/clients/polly'; +const Stream = require('stream'); +const fs = require('fs'); +const path = require('path'); + +export default class LexVoiceClient extends BotClient { + + private botName: string; + private botAlias: string; + private userId: string; + private lastResponse: any; + private sessionAttributes: any; + private props: any; + private lex: LexRuntime; + private polly: Polly; + + constructor(botContext: any, userContext: any) { + super(botContext, userContext); + this.botName = this.botContext.botName; + this.botAlias = this.botContext.botAlias; + this.userId = `${this.userContext.userId}-${Date.now()}`; + this.lastResponse = null; + this.sessionAttributes = this.userContext.userAttributes; + + this.props = { + region: this.botContext.region, + }; + // Optional Auth Environment Variables + this.props.accessKeyId = process.env.chatpickle_access_id || undefined; + this.props.secretAccessKey = process.env.chatpickle_access_secret || undefined; + + this.lex = new LexRuntime(this.props); + + const pollyParams = { + accessKey: process.env.chatpickle_access_id || undefined, + secretAccessKey: process.env.chatpickle_access_secret || undefined, + signatureVersion: "v4", + region: 'us-east-1' + } + this.polly = new Polly(pollyParams); + + console.log(`[${this.userId}] New Conversation with ${this.botName}`); + } + + public async speak(inputText: string): Promise { + + console.log(`[${this.userId}] User: ${inputText}`); + + let audioStream: Buffer = null; + + if (inputText && inputText.indexOf('.pcm') >= 0) { + + const fileName = inputText.trim(); + try { + audioStream = fs.readFileSync('./examples/lexVoice/chatpickle/UserAudioResponses/' + fileName); + } catch (error) { + console.log("Exception occurred while reading audio file: " + error.message); + } + + } else { + + if (inputText === null) { + inputText = ''; + } + + let pollyParams = { + 'Text': inputText, + 'OutputFormat': 'pcm', + 'VoiceId': 'Joanna' + }; + + try { + const data = await this.polly.synthesizeSpeech(pollyParams).promise(); + if (data) { + if (data.AudioStream instanceof Buffer) { + audioStream = data.AudioStream; + } + } + } catch (err) { + console.log('Unexpected error while synthesizing polly speech ' + err.code); + } + } + + console.log('Calling bot with inputText: ' + inputText); + + const params = { + botName: this.botName, + botAlias: this.botAlias, + userId: this.userId, + sessionAttributes: this.sessionAttributes, + contentType: 'audio/x-l16; sample-rate=16000; channel-count=1', + inputStream: audioStream, + accept: 'text/plain; charset=utf-8' + }; + + + try { + this.lastResponse = await this.lex.postContent(params).promise(); + } catch (err) { + console.log(err, err.stack); + } + + let reply: string = null; + + if (this.lastResponse) { + this.sessionAttributes = this.lastResponse ? this.lastResponse.sessionAttributes : null; + reply = this.lastResponse.message ? this.lastResponse.message.trim() : null; + console.log(`[${this.userId}] Bot: ${reply}`); + } + + return reply; + } + + public async fetch(attributePath: string): Promise { + return await get(this.lastResponse, attributePath); + } +}