From 1fd9c82bd8abd80846b85e76f5dcf452878b9e9b Mon Sep 17 00:00:00 2001 From: John Thompson Date: Thu, 27 Jan 2022 14:16:10 -0500 Subject: [PATCH 1/3] Initial code to visualize static data --- src/pages/Demo.tsx | 12 ++++- src/views/ChartView.tsx | 106 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 src/views/ChartView.tsx diff --git a/src/pages/Demo.tsx b/src/pages/Demo.tsx index b7e8b80..7058ec3 100644 --- a/src/pages/Demo.tsx +++ b/src/pages/Demo.tsx @@ -5,11 +5,13 @@ import React, { ChangeEvent } from 'react' import { PlaybackState, SonificationLevel } from '../sonification/SonificationConstants' import { ImportView } from '../views/ImportView' import { DataView } from '../views/DataView' +import { ChartView } from '../views/ChartView' import { FormControl, InputLabel, Select, SelectChangeEvent, MenuItem, Grid, NativeSelect } from '@mui/material' import { DataManager } from '../DataManager' + import { IDemoView } from '../views/demos/IDemoView' import { DemoSimple } from '../views/demos/DemoSimple' import { DemoHighlightRegion } from '../views/demos/DemoHighlightRegion' @@ -19,7 +21,11 @@ import { ExperimentalDemoHighlightRegion } from '../views/demos/ExperimentalDemo const DEMO_VIEW_MAP = { simple: { value: 'simple', label: 'Simple sonification', component: DemoSimple }, highlightRegion: { value: 'highlightRegion', label: 'Highlight points for region', component: DemoHighlightRegion }, - experimentalHighlightRegion: { value: 'experimentalHighlightRegion', label: 'experimental implementation of highlight points for region', component: ExperimentalDemoHighlightRegion }, + experimentalHighlightRegion: { + value: 'experimentalHighlightRegion', + label: 'experimental implementation of highlight points for region', + component: ExperimentalDemoHighlightRegion, + }, } let demoViewRef: React.RefObject | DemoHighlightRegion> = React.createRef() @@ -63,6 +69,10 @@ export class Demo extends React.Component { +
+ +
+
diff --git a/src/views/ChartView.tsx b/src/views/ChartView.tsx new file mode 100644 index 0000000..7e62ebb --- /dev/null +++ b/src/views/ChartView.tsx @@ -0,0 +1,106 @@ +import React from 'react' + +import { DataManager } from '../DataManager' +import * as d3 from 'd3' + +export interface ChartViewState { + rows: any[] + domainX: number[] + domainY: number[] + margin: any + width: number + height: number + fieldY: string[] + fieldX: string +} + +export interface ChartViewProps {} + +export class ChartView extends React.Component { + constructor(props: ChartViewProps) { + super(props) + this.state = { + rows: [], + domainX: [-10, 10], + domainY: [-10, 10], + margin: { l: 20, r: 20, b: 20, t: 20 }, + width: 1340, + height: 500, + fieldX: 'Time', + fieldY: ['Value'], + } + + DataManager.getInstance().addListener(this.handleDataUpdate) + } + + // Private parameters + private scaleX = d3.scaleLinear() + private scaleY = d3.scaleLinear() + + public render() { + const { rows, width, height, margin, domainX, domainY, fieldX, fieldY } = this.state + + this.scaleX.domain(domainX).range([0, width - margin.l - margin.r]) + this.scaleY.domain(domainY).range([height - margin.t - margin.b, 0]) + + const ticksMajorY = this.scaleY.ticks().map((t, i) => { + const y0 = this.scaleY(t) + return ( + + + {t} + + ) + }) + + const ticksMajorX = this.scaleX.ticks().map((t, i) => { + const x0 = this.scaleX(t) + return ( + + + {t} + + ) + }) + + const paths = fieldY.map((f, i) => { + const line = d3 + .line() + .x((d) => this.scaleX(d[fieldX])) + .y((d) => this.scaleY(d[f])) + return + }) + + return ( +
+ + + + {ticksMajorX} + {ticksMajorY} + + {paths} + + +
+ ) + } + + public handleDataUpdate = (table: any): void => { + const { fieldX } = this.state + // const columns = table + // .columnNames() + // .map((c) => ({ field: c, headerName: c, width: 160, renderHeader: (params: any) => {c} })) + const rows = table.objects().map((o, i) => Object.assign({ id: i }, o)), + domainX = d3.extent(rows, (d) => d[fieldX]), + domainY = d3.extent(rows, (d) => d['Value']) + + this.setState({ rows, domainX, domainY }) + } +} From f10f65e66578634499f38ea35debcd3d72ef9a7d Mon Sep 17 00:00:00 2001 From: John Thompson Date: Mon, 31 Jan 2022 15:05:19 -0500 Subject: [PATCH 2/3] Handle window resize for ChartView --- src/views/ChartView.tsx | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/views/ChartView.tsx b/src/views/ChartView.tsx index 7e62ebb..79c28b4 100644 --- a/src/views/ChartView.tsx +++ b/src/views/ChartView.tsx @@ -2,6 +2,7 @@ import React from 'react' import { DataManager } from '../DataManager' import * as d3 from 'd3' +import { YoutubeSearchedForOutlined } from '@mui/icons-material' export interface ChartViewState { rows: any[] @@ -36,6 +37,7 @@ export class ChartView extends React.Component { // Private parameters private scaleX = d3.scaleLinear() private scaleY = d3.scaleLinear() + private chartContainer: React.RefObject = React.createRef() public render() { const { rows, width, height, margin, domainX, domainY, fieldX, fieldY } = this.state @@ -51,7 +53,7 @@ export class ChartView extends React.Component { d={`M0,${y0}h${this.scaleX.range()[1]}`} style={{ stroke: '#999', shapeRendering: 'crispEdges' }} /> - {t} + {t} ) }) @@ -64,7 +66,7 @@ export class ChartView extends React.Component { d={`M${x0},0v${this.scaleY.range()[0]}`} style={{ stroke: '#999', shapeRendering: 'crispEdges' }} /> - {t} + {t} ) }) @@ -78,7 +80,7 @@ export class ChartView extends React.Component { }) return ( -
+
@@ -92,6 +94,19 @@ export class ChartView extends React.Component { ) } + public componentDidMount() { + this.handleResize(); + window.addEventListener('resize', this.handleResize); + } + + public handleResize = (): void => { + if (this.chartContainer && this.chartContainer.current) { + const dimensions = this.chartContainer.current.getBoundingClientRect(), + {width, height} = dimensions; + this.setState({width, height}); + } + } + public handleDataUpdate = (table: any): void => { const { fieldX } = this.state // const columns = table From 568ebafe8fbc7daae433c2876dfefe23a41f0912 Mon Sep 17 00:00:00 2001 From: John Thompson Date: Tue, 15 Feb 2022 16:00:17 -0500 Subject: [PATCH 3/3] Show visualization of static data, update point on chart while playing sonification --- src/pages/Demo.tsx | 8 +- src/sonification/Datum.ts | 6 +- src/sonification/OutputEngine.ts | 10 +- src/views/ChartView.tsx | 137 +++++++++++++++--- src/views/ImportView.tsx | 6 + src/views/demos/DemoHighlightRegion.tsx | 2 +- src/views/demos/DemoSimple.tsx | 2 +- .../demos/ExperimentalDemoHighlightRegion.tsx | 2 +- 8 files changed, 149 insertions(+), 24 deletions(-) diff --git a/src/pages/Demo.tsx b/src/pages/Demo.tsx index a9690c2..a5547ad 100644 --- a/src/pages/Demo.tsx +++ b/src/pages/Demo.tsx @@ -25,6 +25,7 @@ const DEMO_VIEW_MAP = { } let demoViewRef: React.RefObject | DemoHighlightRegion> = React.createRef() +let chartViewRef: React.RefObject = React.createRef() export interface DemoState { dataSummary: any columnList: string[] @@ -66,7 +67,7 @@ export class Demo extends React.Component {
- +
@@ -195,6 +196,11 @@ export class Demo extends React.Component { } } } + // Need to to create one handler for visual + const sinks = outputEngineInstance.getSinks(); + sinks.forEach(sink => { + if (chartViewRef.current) chartViewRef.current.handleSinkUpdate(sink); + }) } } diff --git a/src/sonification/Datum.ts b/src/sonification/Datum.ts index 9cf8e72..c6b90cf 100644 --- a/src/sonification/Datum.ts +++ b/src/sonification/Datum.ts @@ -16,18 +16,20 @@ import * as d3 from 'd3' export class Datum { value: number adjustedValue: number + id: number sinkId: number time: number - constructor(sinkId: number, value: number, time?: number) { + constructor(sinkId: number, value: number, id: number, time?: number) { this.value = value this.adjustedValue = value + this.id = id this.sinkId = sinkId if (time) this.time = time else this.time = d3.now() } public toString(): string { - return `(raw: ${this.value}; adjusted: ${this.adjustedValue}, ${this.time})` + return `(raw: ${this.value}; adjusted: ${this.adjustedValue}; id: ${this.id}, ${this.time})` } } diff --git a/src/sonification/OutputEngine.ts b/src/sonification/OutputEngine.ts index d5fc87a..0370ed4 100644 --- a/src/sonification/OutputEngine.ts +++ b/src/sonification/OutputEngine.ts @@ -67,6 +67,14 @@ export class OutputEngine { return true } + /** + * + * @returns Array of active DataSinks + */ + public getSinks(): DataSink[] { + return [...this.sinks.values()] + } + /** * Get a sink given an Id. Throws an error of sinkId doesn't exist. * @param sinkId @@ -205,7 +213,7 @@ export class OutputEngine { let sink = this.sinks.get(sinkId) if (!sink) throw new Error(`no sink associated with ${sinkId}`) if (DEBUG) console.log(`Sink ${sink}`) - let datum = new Datum(sinkId, point) + let datum = new Datum(sinkId, point, point) switch (this.outputState) { case OutputState.Stopped: // ignore the point if (DEBUG) console.log(`playback: Stopped`) diff --git a/src/views/ChartView.tsx b/src/views/ChartView.tsx index 79c28b4..a288584 100644 --- a/src/views/ChartView.tsx +++ b/src/views/ChartView.tsx @@ -2,10 +2,16 @@ import React from 'react' import { DataManager } from '../DataManager' import * as d3 from 'd3' -import { YoutubeSearchedForOutlined } from '@mui/icons-material' +import { DataSink } from '../sonification/DataSink' +import { Datum } from '../sonification/Datum' +import { DataHandler } from '../sonification/handler/DataHandler' +import { DatumOutput } from '../sonification/output/DatumOutput' +import { ThreeDRotationSharp } from '@mui/icons-material' +// import { Sonifier } from '../sonification/Sonifier' export interface ChartViewState { rows: any[] + valuesX: number[] domainX: number[] domainY: number[] margin: any @@ -13,6 +19,7 @@ export interface ChartViewState { height: number fieldY: string[] fieldX: string + highlight: any[] } export interface ChartViewProps {} @@ -22,6 +29,7 @@ export class ChartView extends React.Component { super(props) this.state = { rows: [], + valuesX: [], domainX: [-10, 10], domainY: [-10, 10], margin: { l: 20, r: 20, b: 20, t: 20 }, @@ -29,21 +37,25 @@ export class ChartView extends React.Component { height: 500, fieldX: 'Time', fieldY: ['Value'], + highlight: [], } + // Sonifier.getSonifierInstance().addPlaybackListener(this.handleDatumUpdate) + DataManager.getInstance().addListener(this.handleDataUpdate) } // Private parameters private scaleX = d3.scaleLinear() private scaleY = d3.scaleLinear() + + private dataHandler: ChartDataHandler | undefined + private chartContainer: React.RefObject = React.createRef() + private pointHighlight: React.RefObject = React.createRef() public render() { - const { rows, width, height, margin, domainX, domainY, fieldX, fieldY } = this.state - - this.scaleX.domain(domainX).range([0, width - margin.l - margin.r]) - this.scaleY.domain(domainY).range([height - margin.t - margin.b, 0]) + const { rows, margin, fieldX, fieldY } = this.state const ticksMajorY = this.scaleY.ticks().map((t, i) => { const y0 = this.scaleY(t) @@ -53,7 +65,9 @@ export class ChartView extends React.Component { d={`M0,${y0}h${this.scaleX.range()[1]}`} style={{ stroke: '#999', shapeRendering: 'crispEdges' }} /> - {t} + + {t} + ) }) @@ -66,7 +80,9 @@ export class ChartView extends React.Component { d={`M${x0},0v${this.scaleY.range()[0]}`} style={{ stroke: '#999', shapeRendering: 'crispEdges' }} /> - {t} + + {t} + ) }) @@ -81,41 +97,128 @@ export class ChartView extends React.Component { return (
- + {ticksMajorX} {ticksMajorY} {paths} + + + + +
) } + private handleMouseMove = (e: React.MouseEvent): void => { + const { rows, valuesX, fieldX, fieldY } = this.state + + if (valuesX.length > 0) { + const mx = d3.pointer(e)[0], + i = d3.bisectCenter(valuesX, this.scaleX.invert(mx)), + tx = this.scaleX(valuesX[i]), + ty = this.scaleY(rows[i][fieldY[0]]) + + if (this.pointHighlight.current) { + d3.select(this.pointHighlight.current).attr('transform', `translate(${tx},${ty})`) + } + } + } + public componentDidMount() { - this.handleResize(); - window.addEventListener('resize', this.handleResize); + this.handleResize() + window.addEventListener('resize', this.handleResize) } public handleResize = (): void => { - if (this.chartContainer && this.chartContainer.current) { + if (this.chartContainer.current) { const dimensions = this.chartContainer.current.getBoundingClientRect(), - {width, height} = dimensions; - this.setState({width, height}); + { width, height } = dimensions + + this.scaleX.range([0, width - this.state.margin.l - this.state.margin.r]) + this.scaleY.range([height - this.state.margin.t - this.state.margin.b, 0]) + + this.setState({ width, height }) + } + } + + public handleSinkUpdate = (sink: DataSink): void => { + if (!this.dataHandler) { + this.dataHandler = new ChartDataHandler(sink, undefined, this.handleDatumUpdate) + sink.addDataHandler(this.dataHandler) } } public handleDataUpdate = (table: any): void => { const { fieldX } = this.state - // const columns = table - // .columnNames() - // .map((c) => ({ field: c, headerName: c, width: 160, renderHeader: (params: any) => {c} })) + const rows = table.objects().map((o, i) => Object.assign({ id: i }, o)), + valuesX = rows.map((d) => d[fieldX]), domainX = d3.extent(rows, (d) => d[fieldX]), domainY = d3.extent(rows, (d) => d['Value']) - this.setState({ rows, domainX, domainY }) + this.scaleX.domain(domainX) + this.scaleY.domain(domainY) + + this.setState({ rows, valuesX, domainX, domainY }) + } + + /** + * Stores relevant information. Value is derived from point.scaledValue. + * @param datum The raw datum + */ + public handleDatumUpdate = (datum?: Datum): void => { + if (datum) { + const { rows, valuesX, fieldX, fieldY } = this.state + + const i = datum.id + + const tx = this.scaleX(valuesX[i]), + ty = this.scaleY(rows[i][fieldY[0]]) + + if (this.pointHighlight.current) { + d3.select(this.pointHighlight.current).attr('transform', `translate(${tx},${ty})`) + } + } + } +} + +// Does it make sense to create a subclass for highlighting a point? That way there could be multiple? +class ChartDataHandler extends DataHandler { + private handleDatumCallback?: (datum?: Datum) => void + + /** + * + * @param sink. DataSink that is providing data to this Handler. + * @param output An optional way to output the data + */ + constructor(sink?: DataSink, output?: DatumOutput, handleDatumCallback?: (datum?: Datum) => void) { + super(sink, output) + this.handleDatumCallback = handleDatumCallback + } + + /** + * Adjusts the value for datum by scaling it to the range [min, max] + * Alternatively, if datum is empty, no need to adjust since the stream is empty + * at this point in time. + * + * @param datum + * @returns Always returns true + */ + handleDatum(datum?: Datum): boolean { + if (!datum) return true + + if (this.handleDatumCallback) this.handleDatumCallback(datum) + + return super.handleDatum(datum) + } + + public toString(): string { + return `ChartDataHandler` } } diff --git a/src/views/ImportView.tsx b/src/views/ImportView.tsx index 5984b3a..45ae4fe 100644 --- a/src/views/ImportView.tsx +++ b/src/views/ImportView.tsx @@ -254,6 +254,12 @@ export class ImportView extends React.Component) => { diff --git a/src/views/demos/DemoHighlightRegion.tsx b/src/views/demos/DemoHighlightRegion.tsx index 418fd80..9b59e94 100644 --- a/src/views/demos/DemoHighlightRegion.tsx +++ b/src/views/demos/DemoHighlightRegion.tsx @@ -32,7 +32,7 @@ export class DemoHighlightRegion maxValue: this.props.dataSummary.max, } } - + public render() { const { minValue, maxValue } = this.state diff --git a/src/views/demos/DemoSimple.tsx b/src/views/demos/DemoSimple.tsx index eca088d..eb6baf6 100644 --- a/src/views/demos/DemoSimple.tsx +++ b/src/views/demos/DemoSimple.tsx @@ -80,7 +80,7 @@ export class DemoSimple let id = this.sink ? this.sink.id : 0 let source = timer(0, 200).pipe( - map((val) => new Datum(id, data[val])), + map((val) => new Datum(id, data[val], val)), take(data.length), ) console.log('setStream in demo') diff --git a/src/views/demos/ExperimentalDemoHighlightRegion.tsx b/src/views/demos/ExperimentalDemoHighlightRegion.tsx index 5eb9b9c..090dcc6 100644 --- a/src/views/demos/ExperimentalDemoHighlightRegion.tsx +++ b/src/views/demos/ExperimentalDemoHighlightRegion.tsx @@ -110,7 +110,7 @@ export class ExperimentalDemoHighlightRegion new Datum(id, data[val])), + map((val) => new Datum(id, data[val], val)), take(data.length), ) console.log('setStream in demo')