Skip to content

Added a new parameter to the julijus constructor that allows direct stream channeling #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aremumahmud wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added a new parameter to the julijus constructor that allows direct stream channeling #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,18 @@ By default, phonemes are defined in `voxforge/hmmdefs`, though you might find [o

### Configuring the engine

The `Julius` constructor takes three arguments which can be used to tune the engine:
The `Julius` constructor takes three to four arguments, with which the stream parameter is optional which can be used to tune the engine:

```js
new Julius('path/to/dfa', 'path/to/dict', options)
```

or the stream object is passed for more control, direct stream channeling and fine tuning of Julius which removes the need of accesing the microphone twice when the need to record while transcribing arises.

```js
new Julius('path/to/dfa', 'path/to/dict', options , stream)
```

_Both 'path/to/dfa' and 'path/to/dict' must be set to use a custom grammar_

##### 'path/to/dfa'
Expand All @@ -114,6 +120,11 @@ _Both 'path/to/dfa' and 'path/to/dict' must be set to use a custom grammar_
- A reference to available options can be found in the [JuliusBook](http://julius.sourceforge.jp/juliusbook/en/).
- Currently, the only supported hidden markov model is from voxforge. The `h` and `hlist` options are unsupported.

##### stream
- a valid stream object gotten from the `navigator.mediaDevices.getUserMedia` method
- if stream is not an audio stream julius will terminate
- if stream doesnt exist julius will terminate

## Examples

### Voice Command
Expand Down
251 changes: 150 additions & 101 deletions dist/julius.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,122 +1,171 @@
(function(window, navigator, undefined) {
var postBuffer = function() {
var that = this;
var that = this;

return function(e) {
var buffer = e.inputBuffer.getChannelData(0);

if (that.audio._transfer) {
var out = e.outputBuffer.getChannelData(0);
return function(e) {
var buffer = e.inputBuffer.getChannelData(0);

for (var i = 0; i < 4096; i++) {
out[i] = buffer[i];
}
}
if (that.audio._transfer) {
var out = e.outputBuffer.getChannelData(0);

for (var i = 0; i < 4096; i++) {
out[i] = buffer[i];
}
}

// Transfer audio to the recognizer
that.recognizer.postMessage(buffer);
};
// Transfer audio to the recognizer
that.recognizer.postMessage(buffer);
};
};

var initializeAudio = function(audio) {
audio.context = new (window.AudioContext || window.webkitAudioContext)();
audio.processor = audio.context.createScriptProcessor(4096, 1, 1);
audio.context = new(window.AudioContext || window.webkitAudioContext)();
audio.processor = audio.context.createScriptProcessor(4096, 1, 1);
};

var bootstrap = function(pathToDfa, pathToDict, options) {
var audio = this.audio;
var recognizer = this.recognizer;
var terminate = this.terminate;
// Compatibility
navigator.getUserMedia = navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia;

navigator.getUserMedia(
{ audio: true },
function(stream) {
audio.source = audio.context.createMediaStreamSource(stream);
audio.source.connect(audio.processor);
audio.processor.connect(audio.context.destination);

// Bootstrap the recognizer
recognizer.postMessage({
type: 'begin',
pathToDfa: pathToDfa,
pathToDict: pathToDict,
options: options
});
},
function(err) {
terminate();
console.error('JuliusJS failed: could not capture microphone input.');
}
);
var audio = this.audio;
var recognizer = this.recognizer;
var terminate = this.terminate;

// Compatibility
navigator.getUserMedia =
navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia;

navigator.getUserMedia({ audio: true },
function(stream) {
audio.source = audio.context.createMediaStreamSource(stream);
audio.source.connect(audio.processor);
audio.processor.connect(audio.context.destination);

// Bootstrap the recognizer
recognizer.postMessage({
type: "begin",
pathToDfa: pathToDfa,
pathToDict: pathToDict,
options: options,
});
},
function(err) {
terminate();
console.error("JuliusJS failed: could not capture microphone input.");
}
);
};

var Julius = function(pathToDfa, pathToDict, options) {
var that = this;
options = options || {};

// The context's nodemap: `source` -> `processor` -> `destination`
this.audio = {
// `AudioContext`
context: null,
// `AudioSourceNode` from captured microphone input
source: null,
// `ScriptProcessorNode` for julius
processor: null,
_transfer: options.transfer
};

// Do not pollute the object
delete options.transfer;

// _Recognition is offloaded to a separate thread to avoid slowing UI_
this.recognizer = new Worker(options.pathToWorker || 'worker.js');

this.recognizer.onmessage = function(e) {
if (e.data.type === 'begin') {
that.audio.processor.onaudioprocess = postBuffer.call(that);

} else if (e.data.type === 'recog') {
if (e.data.firstpass) {
typeof that.onfirstpass === 'function' &&
that.onfirstpass(e.data.sentence, e.data.score);
} else
typeof that.onrecognition === 'function' &&
that.onrecognition(e.data.sentence);

} else if (e.data.type === 'log') {
typeof that.onlog === 'function' &&
that.onlog(e.data.sentence);

} else if (e.data.type === 'error') {
console.error(e.data.error);
that.terminate();

} else {
console.info('Unexpected data received from julius:');
console.info(e.data);
//This bootstrapToStream() function implements direct stream channeling and allows developers to have more control over the bootstrap function and avoid accessing the microphone twice
//which could be inefficient and annoying on mobile devices when trying to recognize and record at the same time

var bootstrapToStream = function(pathToDfa, pathToDict, options, stream) {
//check if stream object exists
if (!stream) {
terminate();
return console.error("JuliusJS failed: Stream Object is undefined.");
}

//check if the stream accepts audio i.e has an audio track
const hasAudioTrack = stream.getAudioTracks().length > 0;

if (!hasAudioTrack) {
terminate();
return console.error(
"JuliusJS failed: Stream Object provided is not an audio stream"
);
}
};

initializeAudio(this.audio);
bootstrap.call(this, pathToDfa, pathToDict, options);
var audio = this.audio;
var recognizer = this.recognizer;
var terminate = this.terminate;

try {
audio.source = audio.context.createMediaStreamSource(stream);
audio.source.connect(audio.processor);
audio.processor.connect(audio.context.destination);

// Bootstrap the recognizer
recognizer.postMessage({
type: "begin",
pathToDfa: pathToDfa,
pathToDict: pathToDict,
options: options,
});
} catch (err) {
terminate();
console.error("JuliusJS failed: an unexpected error occured.", err);
}
};

var Julius = function(pathToDfa, pathToDict, options, stream) {
var that = this;
options = options || {};

// The context's nodemap: `source` -> `processor` -> `destination`
this.audio = {
// `AudioContext`
context: null,
// `AudioSourceNode` from captured microphone input
source: null,
// `ScriptProcessorNode` for julius
processor: null,
_transfer: options.transfer,
};

// Do not pollute the object
delete options.transfer;

// _Recognition is offloaded to a separate thread to avoid slowing UI_
this.recognizer = new Worker(options.pathToWorker || "worker.js");

this.recognizer.onmessage = function(e) {
if (e.data.type === "begin") {
that.audio.processor.onaudioprocess = postBuffer.call(that);
} else if (e.data.type === "recog") {
if (e.data.firstpass) {
typeof that.onfirstpass === "function" &&
that.onfirstpass(e.data.sentence, e.data.score);
} else
typeof that.onrecognition === "function" &&
that.onrecognition(e.data.sentence);
} else if (e.data.type === "log") {
typeof that.onlog === "function" && that.onlog(e.data.sentence);
} else if (e.data.type === "error") {
console.error(e.data.error);
that.terminate();
} else {
console.info("Unexpected data received from julius:");
console.info(e.data);
}
};

initializeAudio(this.audio);

stream
?
bootstrapToStream.call(this, pathToDfa, pathToDict, options, stream) :
bootstrap.call(this, pathToDfa, pathToDict, options);
};

Julius.prototype.onfirstpass = function(sentence) { /* noop */ };
Julius.prototype.onrecognition = function(sentence, score) { /* noop */ };
Julius.prototype.onlog = function(obj) { console.log(obj); };
Julius.prototype.onfail = function() { /* noop */ };
Julius.prototype.onfirstpass = function(sentence) {
/* noop */
};
Julius.prototype.onrecognition = function(sentence, score) {
/* noop */
};
Julius.prototype.onlog = function(obj) {
console.log(obj);
};
Julius.prototype.onfail = function() {
/* noop */
};
Julius.prototype.terminate = function(cb) {
this.audio.processor.onaudioprocess = null;
this.recognizer.terminate();
console.error('JuliusJS was terminated.');
typeof this.onfail === 'function' && this.onfail();
this.audio.processor.onaudioprocess = null;
this.recognizer.terminate();
console.error("JuliusJS was terminated.");
typeof this.onfail === "function" && this.onfail();
};

window.Julius = Julius;
}(window,window.navigator) );
})(window, window.navigator);